Home /Modules/test-suites/regression_support.py

regression_support.py

test-suites/regression/python/regression_support.py · 308 line(s) · py

"""Shared plumbing for the SLeeLa regression set (Python layer).

* locates the repository, the toolchain and the SLeeLa compiler binary;
* builds the C and C++ regression binaries (strict warnings, -Werror, with the
  instrumented allocator linked in) and runs them;
* provides helpers to run `sleela` hermetically (own scratch directory, fast
  integrity manifest) and to filter its banner noise;
* records KNOWN ISSUES: contracts the code is expected to honour but does not.
  They are reported, never silently dropped, and never fail the run.

Standard library only.
"""
from __future__ import annotations

import hashlib
import json
import os
import re
import shutil
import subprocess
import tempfile
import threading
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Sequence

HERE = Path(__file__).resolve().parent
REG = HERE.parent                      # test-suites/regression
ROOT = REG.parents[1]                  # repository root
BUILD = REG / "build"                  # ignored by the repository's **/build/ rule
LOGS = BUILD / "logs"
# per-process scratch: two concurrent runs must not delete each other's working files
SCRATCH = BUILD / "scratch" / f"run-{os.getpid()}"

CORE = ROOT / "impl" / "core"
FRONTEND = ROOT / "impl" / "frontend"

CC = os.environ.get("CC", "cc")
CXX = os.environ.get("CXX", "c++")

# The project's own audit flags (test-suites/run-all.sh) for the test TUs.
STRICT_C = ["-std=c11", "-O1", "-g", "-Wall", "-Wextra", "-Wpedantic", "-Wconversion",
            "-Wshadow", "-Wformat=2", "-Werror"]
STRICT_CXX = ["-std=c++17", "-O1", "-g", "-Wall", "-Wextra", "-Wpedantic", "-Wconversion",
              "-Wshadow", "-Wformat=2", "-Werror"]
# Code under test is compiled as the project builds it: warnings on, not fatal.
LIB_C = ["-std=gnu11", "-O1", "-g", "-Wall"]
LIB_CXX = ["-std=c++17", "-O1", "-g", "-Wall"]
WRAP = ["-Wl,--wrap=malloc", "-Wl,--wrap=calloc", "-Wl,--wrap=realloc", "-Wl,--wrap=free"]

# logical name -> source file of the code under test
SOURCES: Dict[str, Path] = {
    "utf16": CORE / "sleela_utf16.c",
    "java_string": CORE / "sleela_java_string.c",
    "java_string_codec": CORE / "sleela_java_string_codec.c",
    "regex": CORE / "sleela_regex.c",
    "path": CORE / "sleela_path.c",
    "time": CORE / "sleela_time.c",
    "memory": CORE / "sleela_memory.c",
    "events": CORE / "sleela_events.c",
    "memmgr": CORE / "sleela_memmgr.c",
    "thread": CORE / "sleela_thread.c",
    "net": CORE / "sleela_net.c",
    "synchro": CORE / "sleela_synchro.c",
    "http6": ROOT / "http-6.0" / "http6_protocol.c",
    "router": ROOT / "preferred-routers" / "preferred_router.c",
    "lexer": FRONTEND / "lexer.cpp",
}

# suite -> (language, [code-under-test objects it links])
SUITES: Dict[str, tuple] = {
    "reg_utf16": ("c", ["utf16"]),
    "reg_java_string": ("c", ["utf16", "java_string"]),
    "reg_java_string_codec": ("c", ["java_string", "java_string_codec"]),
    "reg_regex": ("c", ["regex"]),
    "reg_path": ("c", ["path"]),
    "reg_time_memory": ("c", ["time", "memory"]),
    "reg_events": ("c", ["events"]),
    "reg_memmgr": ("c", ["memmgr", "memory", "thread"]),
    "reg_thread": ("c", ["thread"]),
    "reg_net": ("c", ["net"]),
    "reg_synchro": ("c", ["synchro"]),
    "reg_http6": ("c", ["http6", "router"]),
    "reg_router": ("c", ["router"]),
    "reg_lexer": ("cpp", ["lexer"]),
    "reg_model": ("cpp", ["utf16", "java_string", "java_string_codec", "regex", "path"]),
}


class BuildError(RuntimeError):
    pass


def have_toolchain() -> Optional[str]:
    """Return None when a C and C++ compiler are usable, else a reason."""
    for tool in (CC, CXX):
        if shutil.which(tool) is None:
            return f"compiler not found: {tool}"
    return None


def _run(cmd: Sequence[str], cwd: Optional[Path] = None, timeout: int = 600) -> subprocess.CompletedProcess:
    return subprocess.run([str(c) for c in cmd], cwd=str(cwd) if cwd else None, capture_output=True,
                          text=True, timeout=timeout)


_build_lock = threading.Lock()
_built: Dict[str, Path] = {}


def _extra_flags() -> List[str]:
    return ["-fsanitize=address,undefined", "-fno-omit-frame-pointer"] if os.environ.get("REG_SANITIZE") == "1" else []


def compile_object(src: Path, obj: Path, cxx: bool = False, flags: Optional[Iterable[str]] = None) -> Path:
    obj.parent.mkdir(parents=True, exist_ok=True)
    base = LIB_CXX if cxx else LIB_C
    cmd = [CXX if cxx else CC, *(flags if flags is not None else base), *_extra_flags(), "-c", src, "-o", obj]
    r = _run(cmd)
    if r.returncode != 0:
        raise BuildError(f"compiling {src} failed:\n{r.stderr}")
    return obj


def support_object() -> Path:
    key = "support"
    with _build_lock:
        if key not in _built:
            _built[key] = compile_object(REG / "c" / "reg_support.c", BUILD / "obj" / "reg_support.o",
                                         flags=["-std=gnu11", "-O1", "-g", "-Wall", "-Wextra"])
    return _built[key]


def library_object(name: str) -> Path:
    with _build_lock:
        if name not in _built:
            src = SOURCES[name]
            _built[name] = compile_object(src, BUILD / "obj" / f"{name}.o", cxx=src.suffix == ".cpp")
    return _built[name]


def build_suite(name: str, replace: Optional[Dict[str, Path]] = None, tag: str = "") -> Path:
    """Compile and link one regression binary. `replace` swaps in a different
    object file for a logical library (used by the mutation tests)."""
    lang, libs = SUITES[name]
    cxx = lang == "cpp"
    src = REG / lang / f"{name}.{'cpp' if cxx else 'c'}"
    out = BUILD / "bin" / (name + (f".{tag}" if tag else ""))
    out.parent.mkdir(parents=True, exist_ok=True)
    objs = [(replace or {}).get(lib) or library_object(lib) for lib in libs]
    cmd = [CXX if cxx else CC, *(STRICT_CXX if cxx else STRICT_C), *_extra_flags(), src,
           support_object(), *objs, *WRAP, "-pthread", "-o", out]
    if not cxx:
        cmd += ["-lm"]
    r = _run(cmd)
    if r.returncode != 0:
        raise BuildError(f"building {name} failed:\n{r.stderr}")
    return out


@dataclass
class SuiteResult:
    returncode: int
    stdout: str
    stderr: str
    summary: Dict[str, int] = field(default_factory=dict)

    @property
    def failures(self) -> List[str]:
        return [ln for ln in self.stderr.splitlines() if "FAIL" in ln]

    @property
    def known_issues(self) -> List[str]:
        return [ln.strip() for ln in self.stdout.splitlines() if "KNOWN-ISSUE" in ln]


_SUMMARY = re.compile(r"^SUITE (\S+): (.*)$", re.M)


def run_binary(exe: Path, timeout: int = 900) -> SuiteResult:
    tmp = SCRATCH / "c-tmp"
    tmp.mkdir(parents=True, exist_ok=True)
    env = os.environ.copy()
    env["REG_TMPDIR"] = str(tmp)
    r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=timeout, env=env)
    summary: Dict[str, int] = {}
    m = _SUMMARY.search(r.stdout)
    if m:
        for kv in m.group(2).split():
            k, _, v = kv.partition("=")
            if v.isdigit():
                summary[k] = int(v)
    LOGS.mkdir(parents=True, exist_ok=True)
    (LOGS / f"{exe.name}.log").write_text(r.stdout + "\n--- stderr ---\n" + r.stderr)
    return SuiteResult(r.returncode, r.stdout, r.stderr, summary)


# ---------------------------------------------------------------- known issues

_known: List[dict] = []


def record_known_issue(issue_id: str, description: str, still_open: bool, source: str) -> None:
    _known.append({"id": issue_id, "description": description, "open": still_open, "source": source})
    KNOWN_FILE.parent.mkdir(parents=True, exist_ok=True)
    existing = json.loads(KNOWN_FILE.read_text()) if KNOWN_FILE.exists() else []
    existing = [e for e in existing if not (e["id"] == issue_id and e["source"] == source)]
    existing.append(_known[-1])
    KNOWN_FILE.write_text(json.dumps(existing, indent=1, sort_keys=True))


KNOWN_FILE = BUILD / "known-issues.json"


# ------------------------------------------------------------------- sleela CLI

NOISE = ("[memory-manager]", "[security]", "[defender]", "verification:")


class CompilerSelectionError(RuntimeError):
    pass


def sleela_binary() -> Optional[Path]:
    """Compiler under test: $SLEELA_BIN, else the source build, else the
    committed prebuilt binary.

    The result is always an absolute path: the CLI suites run the compiler from
    per-test scratch directories, where a relative path would not resolve.
    An explicit $SLEELA_BIN that is not an executable file is an error, never a
    silent fallback to a different compiler."""
    explicit = os.environ.get("SLEELA_BIN")
    if explicit:
        path = Path(explicit).expanduser().resolve()
        if not (path.is_file() and os.access(path, os.X_OK)):
            raise CompilerSelectionError(f"SLEELA_BIN={explicit!r} is not an executable file (resolved to {path})")
        return path
    for cand in (ROOT / "impl" / "build" / "sleela", ROOT / "working-binaries" / "sleela"):
        if cand.is_file() and os.access(cand, os.X_OK):
            return cand.resolve()
    return None


def sha256_file(path: Path) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as fh:
        for chunk in iter(lambda: fh.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def fast_manifest() -> Path:
    """A tiny integrity manifest (SHEET.sheet + the verifier) so each compiler
    invocation does not re-hash the 18k-file repository. The real manifest and
    the fail-closed behaviour are covered by test_cli_gate / test_integrity."""
    out = BUILD / "fast-manifest.json"
    files = ["SHEET.sheet", "tools/verify-before-execution.py"]
    doc = {"algorithm": "sha256", "files": [{"path": f, "sha256": sha256_file(ROOT / f)} for f in files]}
    BUILD.mkdir(parents=True, exist_ok=True)
    out.write_text(json.dumps(doc, indent=1))
    return out


def cli_env(manifest: Optional[Path] = None, **extra: str) -> Dict[str, str]:
    env = os.environ.copy()
    env.update({
        "SLEELA_HOME": str(ROOT),
        "SLEELA_LIB": str(ROOT / "lib"),
        "SLEELA_SHEET": str(ROOT / "SHEET.sheet"),
        "SLEELA_SHA256_MANIFEST": str(manifest or fast_manifest()),
    })
    env.update(extra)
    return env


def run_sleela(args: Sequence[object], cwd: Path, timeout: int = 120, env: Optional[Dict[str, str]] = None,
               binary: Optional[Path] = None) -> subprocess.CompletedProcess:
    exe = binary or sleela_binary()
    if exe is None:
        raise RuntimeError("no sleela binary available")
    return subprocess.run([str(exe), *[str(a) for a in args]], cwd=str(cwd), capture_output=True, text=True,
                          encoding="utf-8", errors="replace", timeout=timeout, env=env or cli_env())


def program_output(cp: subprocess.CompletedProcess) -> List[str]:
    """Program stdout with the toolchain banner lines removed."""
    return [ln for ln in cp.stdout.splitlines() if not ln.startswith(NOISE)]


def diagnostics(cp: subprocess.CompletedProcess) -> str:
    """Compiler/runtime diagnostics (stderr) with banner lines removed."""
    return "\n".join(ln for ln in cp.stderr.splitlines() if not ln.startswith(NOISE))


def fresh_dir(label: str) -> Path:
    SCRATCH.mkdir(parents=True, exist_ok=True)
    return Path(tempfile.mkdtemp(prefix=label + "-", dir=str(SCRATCH)))


def write_program(directory: Path, name: str, source: str) -> Path:
    p = directory / name
    p.write_text(source, encoding="utf-8")
    return p


def cleanup_scratch() -> None:
    shutil.rmtree(SCRATCH, ignore_errors=True)