regression_support.py
test-suites/regression/python/regression_support.py · 308 line(s) · py
"""Shared plumbing for the SLeeLa regression set (Python layer).
* locates the repository, the toolchain and the SLeeLa compiler binary;
* builds the C and C++ regression binaries (strict warnings, -Werror, with the
instrumented allocator linked in) and runs them;
* provides helpers to run `sleela` hermetically (own scratch directory, fast
integrity manifest) and to filter its banner noise;
* records KNOWN ISSUES: contracts the code is expected to honour but does not.
They are reported, never silently dropped, and never fail the run.
Standard library only.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
import subprocess
import tempfile
import threading
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Sequence
HERE = Path(__file__).resolve().parent
REG = HERE.parent # test-suites/regression
ROOT = REG.parents[1] # repository root
BUILD = REG / "build" # ignored by the repository's **/build/ rule
LOGS = BUILD / "logs"
# per-process scratch: two concurrent runs must not delete each other's working files
SCRATCH = BUILD / "scratch" / f"run-{os.getpid()}"
CORE = ROOT / "impl" / "core"
FRONTEND = ROOT / "impl" / "frontend"
CC = os.environ.get("CC", "cc")
CXX = os.environ.get("CXX", "c++")
# The project's own audit flags (test-suites/run-all.sh) for the test TUs.
STRICT_C = ["-std=c11", "-O1", "-g", "-Wall", "-Wextra", "-Wpedantic", "-Wconversion",
"-Wshadow", "-Wformat=2", "-Werror"]
STRICT_CXX = ["-std=c++17", "-O1", "-g", "-Wall", "-Wextra", "-Wpedantic", "-Wconversion",
"-Wshadow", "-Wformat=2", "-Werror"]
# Code under test is compiled as the project builds it: warnings on, not fatal.
LIB_C = ["-std=gnu11", "-O1", "-g", "-Wall"]
LIB_CXX = ["-std=c++17", "-O1", "-g", "-Wall"]
WRAP = ["-Wl,--wrap=malloc", "-Wl,--wrap=calloc", "-Wl,--wrap=realloc", "-Wl,--wrap=free"]
# logical name -> source file of the code under test
SOURCES: Dict[str, Path] = {
"utf16": CORE / "sleela_utf16.c",
"java_string": CORE / "sleela_java_string.c",
"java_string_codec": CORE / "sleela_java_string_codec.c",
"regex": CORE / "sleela_regex.c",
"path": CORE / "sleela_path.c",
"time": CORE / "sleela_time.c",
"memory": CORE / "sleela_memory.c",
"events": CORE / "sleela_events.c",
"memmgr": CORE / "sleela_memmgr.c",
"thread": CORE / "sleela_thread.c",
"net": CORE / "sleela_net.c",
"synchro": CORE / "sleela_synchro.c",
"http6": ROOT / "http-6.0" / "http6_protocol.c",
"router": ROOT / "preferred-routers" / "preferred_router.c",
"lexer": FRONTEND / "lexer.cpp",
}
# suite -> (language, [code-under-test objects it links])
SUITES: Dict[str, tuple] = {
"reg_utf16": ("c", ["utf16"]),
"reg_java_string": ("c", ["utf16", "java_string"]),
"reg_java_string_codec": ("c", ["java_string", "java_string_codec"]),
"reg_regex": ("c", ["regex"]),
"reg_path": ("c", ["path"]),
"reg_time_memory": ("c", ["time", "memory"]),
"reg_events": ("c", ["events"]),
"reg_memmgr": ("c", ["memmgr", "memory", "thread"]),
"reg_thread": ("c", ["thread"]),
"reg_net": ("c", ["net"]),
"reg_synchro": ("c", ["synchro"]),
"reg_http6": ("c", ["http6", "router"]),
"reg_router": ("c", ["router"]),
"reg_lexer": ("cpp", ["lexer"]),
"reg_model": ("cpp", ["utf16", "java_string", "java_string_codec", "regex", "path"]),
}
class BuildError(RuntimeError):
pass
def have_toolchain() -> Optional[str]:
"""Return None when a C and C++ compiler are usable, else a reason."""
for tool in (CC, CXX):
if shutil.which(tool) is None:
return f"compiler not found: {tool}"
return None
def _run(cmd: Sequence[str], cwd: Optional[Path] = None, timeout: int = 600) -> subprocess.CompletedProcess:
return subprocess.run([str(c) for c in cmd], cwd=str(cwd) if cwd else None, capture_output=True,
text=True, timeout=timeout)
_build_lock = threading.Lock()
_built: Dict[str, Path] = {}
def _extra_flags() -> List[str]:
return ["-fsanitize=address,undefined", "-fno-omit-frame-pointer"] if os.environ.get("REG_SANITIZE") == "1" else []
def compile_object(src: Path, obj: Path, cxx: bool = False, flags: Optional[Iterable[str]] = None) -> Path:
obj.parent.mkdir(parents=True, exist_ok=True)
base = LIB_CXX if cxx else LIB_C
cmd = [CXX if cxx else CC, *(flags if flags is not None else base), *_extra_flags(), "-c", src, "-o", obj]
r = _run(cmd)
if r.returncode != 0:
raise BuildError(f"compiling {src} failed:\n{r.stderr}")
return obj
def support_object() -> Path:
key = "support"
with _build_lock:
if key not in _built:
_built[key] = compile_object(REG / "c" / "reg_support.c", BUILD / "obj" / "reg_support.o",
flags=["-std=gnu11", "-O1", "-g", "-Wall", "-Wextra"])
return _built[key]
def library_object(name: str) -> Path:
with _build_lock:
if name not in _built:
src = SOURCES[name]
_built[name] = compile_object(src, BUILD / "obj" / f"{name}.o", cxx=src.suffix == ".cpp")
return _built[name]
def build_suite(name: str, replace: Optional[Dict[str, Path]] = None, tag: str = "") -> Path:
"""Compile and link one regression binary. `replace` swaps in a different
object file for a logical library (used by the mutation tests)."""
lang, libs = SUITES[name]
cxx = lang == "cpp"
src = REG / lang / f"{name}.{'cpp' if cxx else 'c'}"
out = BUILD / "bin" / (name + (f".{tag}" if tag else ""))
out.parent.mkdir(parents=True, exist_ok=True)
objs = [(replace or {}).get(lib) or library_object(lib) for lib in libs]
cmd = [CXX if cxx else CC, *(STRICT_CXX if cxx else STRICT_C), *_extra_flags(), src,
support_object(), *objs, *WRAP, "-pthread", "-o", out]
if not cxx:
cmd += ["-lm"]
r = _run(cmd)
if r.returncode != 0:
raise BuildError(f"building {name} failed:\n{r.stderr}")
return out
@dataclass
class SuiteResult:
returncode: int
stdout: str
stderr: str
summary: Dict[str, int] = field(default_factory=dict)
@property
def failures(self) -> List[str]:
return [ln for ln in self.stderr.splitlines() if "FAIL" in ln]
@property
def known_issues(self) -> List[str]:
return [ln.strip() for ln in self.stdout.splitlines() if "KNOWN-ISSUE" in ln]
_SUMMARY = re.compile(r"^SUITE (\S+): (.*)$", re.M)
def run_binary(exe: Path, timeout: int = 900) -> SuiteResult:
tmp = SCRATCH / "c-tmp"
tmp.mkdir(parents=True, exist_ok=True)
env = os.environ.copy()
env["REG_TMPDIR"] = str(tmp)
r = subprocess.run([str(exe)], capture_output=True, text=True, timeout=timeout, env=env)
summary: Dict[str, int] = {}
m = _SUMMARY.search(r.stdout)
if m:
for kv in m.group(2).split():
k, _, v = kv.partition("=")
if v.isdigit():
summary[k] = int(v)
LOGS.mkdir(parents=True, exist_ok=True)
(LOGS / f"{exe.name}.log").write_text(r.stdout + "\n--- stderr ---\n" + r.stderr)
return SuiteResult(r.returncode, r.stdout, r.stderr, summary)
# ---------------------------------------------------------------- known issues
_known: List[dict] = []
def record_known_issue(issue_id: str, description: str, still_open: bool, source: str) -> None:
_known.append({"id": issue_id, "description": description, "open": still_open, "source": source})
KNOWN_FILE.parent.mkdir(parents=True, exist_ok=True)
existing = json.loads(KNOWN_FILE.read_text()) if KNOWN_FILE.exists() else []
existing = [e for e in existing if not (e["id"] == issue_id and e["source"] == source)]
existing.append(_known[-1])
KNOWN_FILE.write_text(json.dumps(existing, indent=1, sort_keys=True))
KNOWN_FILE = BUILD / "known-issues.json"
# ------------------------------------------------------------------- sleela CLI
NOISE = ("[memory-manager]", "[security]", "[defender]", "verification:")
class CompilerSelectionError(RuntimeError):
pass
def sleela_binary() -> Optional[Path]:
"""Compiler under test: $SLEELA_BIN, else the source build, else the
committed prebuilt binary.
The result is always an absolute path: the CLI suites run the compiler from
per-test scratch directories, where a relative path would not resolve.
An explicit $SLEELA_BIN that is not an executable file is an error, never a
silent fallback to a different compiler."""
explicit = os.environ.get("SLEELA_BIN")
if explicit:
path = Path(explicit).expanduser().resolve()
if not (path.is_file() and os.access(path, os.X_OK)):
raise CompilerSelectionError(f"SLEELA_BIN={explicit!r} is not an executable file (resolved to {path})")
return path
for cand in (ROOT / "impl" / "build" / "sleela", ROOT / "working-binaries" / "sleela"):
if cand.is_file() and os.access(cand, os.X_OK):
return cand.resolve()
return None
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as fh:
for chunk in iter(lambda: fh.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def fast_manifest() -> Path:
"""A tiny integrity manifest (SHEET.sheet + the verifier) so each compiler
invocation does not re-hash the 18k-file repository. The real manifest and
the fail-closed behaviour are covered by test_cli_gate / test_integrity."""
out = BUILD / "fast-manifest.json"
files = ["SHEET.sheet", "tools/verify-before-execution.py"]
doc = {"algorithm": "sha256", "files": [{"path": f, "sha256": sha256_file(ROOT / f)} for f in files]}
BUILD.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(doc, indent=1))
return out
def cli_env(manifest: Optional[Path] = None, **extra: str) -> Dict[str, str]:
env = os.environ.copy()
env.update({
"SLEELA_HOME": str(ROOT),
"SLEELA_LIB": str(ROOT / "lib"),
"SLEELA_SHEET": str(ROOT / "SHEET.sheet"),
"SLEELA_SHA256_MANIFEST": str(manifest or fast_manifest()),
})
env.update(extra)
return env
def run_sleela(args: Sequence[object], cwd: Path, timeout: int = 120, env: Optional[Dict[str, str]] = None,
binary: Optional[Path] = None) -> subprocess.CompletedProcess:
exe = binary or sleela_binary()
if exe is None:
raise RuntimeError("no sleela binary available")
return subprocess.run([str(exe), *[str(a) for a in args]], cwd=str(cwd), capture_output=True, text=True,
encoding="utf-8", errors="replace", timeout=timeout, env=env or cli_env())
def program_output(cp: subprocess.CompletedProcess) -> List[str]:
"""Program stdout with the toolchain banner lines removed."""
return [ln for ln in cp.stdout.splitlines() if not ln.startswith(NOISE)]
def diagnostics(cp: subprocess.CompletedProcess) -> str:
"""Compiler/runtime diagnostics (stderr) with banner lines removed."""
return "\n".join(ln for ln in cp.stderr.splitlines() if not ln.startswith(NOISE))
def fresh_dir(label: str) -> Path:
SCRATCH.mkdir(parents=True, exist_ok=True)
return Path(tempfile.mkdtemp(prefix=label + "-", dir=str(SCRATCH)))
def write_program(directory: Path, name: str, source: str) -> Path:
p = directory / name
p.write_text(source, encoding="utf-8")
return p
def cleanup_scratch() -> None:
shutil.rmtree(SCRATCH, ignore_errors=True)