run_regression.py
test-suites/regression/python/run_regression.py · 164 line(s) · py
#!/usr/bin/env python3
"""SLeeLa regression runner (C, C++ and Python suites).
python3 test-suites/regression/python/run_regression.py # everything
python3 test-suites/regression/python/run_regression.py --fast # skip mutation testing
python3 test-suites/regression/python/run_regression.py --only native cli
python3 test-suites/regression/python/run_regression.py --list
Groups
native builds and runs the C and C++ suites (strict flags, leak/heap-guard checked)
ctypes the C core driven from Python against Python oracles
cli the `sleela` compiler: language semantics, contract, transpilers
corpus hostile and malformed inputs (annotations, traversal, HTTP frames, parser, limits)
integrity SHA-256 gate tooling, committed manifest, /lib inventory
mutation injects defects into the code under test; every one must be caught
Exit status 0 only when every test passes. KNOWN ISSUES (defects the suites
have found and that are still open) are listed at the end; they do not change the
exit status. Environment: CC, CXX, SLEELA_BIN, REG_SANITIZE=1 (ASan/UBSan, when
the toolchain ships them), REG_SKIP_MUTATION=1.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
import time
import unittest
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
import regression_support as rs # noqa: E402
GROUPS = {
"native": ["test_native_suites"],
"ctypes": ["test_ctypes_differential"],
"cli": ["test_cli_language", "test_cli_contract", "test_cli_transpilers"],
"corpus": ["test_corpus"],
"integrity": ["test_integrity_tools", "test_doc_versions"],
"mutation": ["test_mutation_kill"],
}
def describe_environment() -> None:
import shutil
import subprocess
print("environment")
print(f" repository : {rs.ROOT}")
for tool in (rs.CC, rs.CXX):
found = shutil.which(tool)
ver = ""
if found:
ver = subprocess.run([found, "--version"], capture_output=True, text=True).stdout.splitlines()[0]
print(f" {tool:<10} : {ver or 'NOT FOUND'}")
binary = rs.sleela_binary()
if binary:
banner = subprocess.run([str(binary), "version"], capture_output=True, text=True, env=rs.cli_env()).stdout
line = next((ln for ln in banner.splitlines() if "SLeeLa" in ln), "?")
print(f" sleela : {binary}\n {line}")
else:
print(" sleela : NOT FOUND (CLI suites will be skipped; build impl/build/sleela)")
print(f" java/javac : {'yes' if shutil.which('javac') and shutil.which('java') else 'no (JVM differential skipped)'}")
print(f" sanitizers : {'ASan+UBSan requested' if os.environ.get('REG_SANITIZE') == '1' else 'off (REG_SANITIZE=1 to enable)'}")
print()
def print_known_issues() -> None:
if not rs.KNOWN_FILE.exists():
return
found = json.loads(rs.KNOWN_FILE.read_text())
merged: dict = {}
for e in found:
cur = merged.setdefault(e["id"], {"open": False, "description": e["description"], "sources": set()})
cur["open"] = cur["open"] or e["open"]
cur["sources"].add(e["source"])
if e["open"] and not e["description"].startswith("KNOWN-ISSUE"):
cur["description"] = e["description"]
if not merged:
return
print("\nKNOWN ISSUES (reported by the suites; they do not fail the run)")
for ident in sorted(merged):
e = merged[ident]
state = "OPEN " if e["open"] else "FIXED"
text = e["description"]
if text.startswith("KNOWN-ISSUE"):
text = text.split(":", 1)[-1].strip()
print(f" [{state}] {ident:<22} {text}")
print(f" found by: {', '.join(sorted(e['sources']))}")
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--only", nargs="+", choices=sorted(GROUPS), help="run only these groups")
ap.add_argument("--fast", action="store_true", help="skip mutation testing")
ap.add_argument("--sanitize", action="store_true", help="build with -fsanitize=address,undefined")
ap.add_argument("--list", action="store_true", help="list tests and exit")
ap.add_argument("-v", "--verbose", action="store_true")
ap.add_argument("-k", dest="pattern", help="only tests whose id contains this text")
ap.add_argument("--clean", action="store_true", help="delete build output and exit")
args = ap.parse_args()
if args.clean:
import shutil
shutil.rmtree(rs.BUILD, ignore_errors=True)
return 0
if args.fast:
os.environ["REG_SKIP_MUTATION"] = "1"
if args.sanitize:
os.environ["REG_SANITIZE"] = "1"
groups = args.only or [g for g in GROUPS if not (args.fast and g == "mutation")]
loader = unittest.TestLoader()
suite = unittest.TestSuite()
for g in groups:
for module in GROUPS[g]:
suite.addTests(loader.loadTestsFromName(module))
def flatten(s):
for t in s:
if isinstance(t, unittest.TestSuite):
yield from flatten(t)
else:
yield t
tests = [t for t in flatten(suite) if not args.pattern or args.pattern in t.id()]
if args.list:
for t in tests:
print(t.id())
print(f"{len(tests)} tests")
return 0
try:
rs.sleela_binary()
except rs.CompilerSelectionError as exc:
print(f"run_regression: {exc}", file=sys.stderr)
return 2
rs.BUILD.mkdir(parents=True, exist_ok=True)
if rs.KNOWN_FILE.exists():
rs.KNOWN_FILE.unlink()
describe_environment()
sys.stdout.flush()
selected = unittest.TestSuite(tests)
started = time.time()
result = unittest.TextTestRunner(verbosity=2 if args.verbose else 1, buffer=False).run(selected)
elapsed = time.time() - started
sys.stderr.flush()
print_known_issues()
rs.cleanup_scratch()
ok = result.wasSuccessful()
print(f"\nREGRESSION {'PASS' if ok else 'FAIL'}: {result.testsRun} tests, {len(result.failures)} failed, "
f"{len(result.errors)} errors, {len(result.skipped)} skipped in {elapsed:.0f}s")
for test, _ in result.failures + result.errors:
print(f" FAILED: {test.id()}")
return 0 if ok else 1
if __name__ == "__main__":
raise SystemExit(main())