Home /Modules/test-suites/run_regression.py

run_regression.py

test-suites/regression/python/run_regression.py · 164 line(s) · py

#!/usr/bin/env python3
"""SLeeLa regression runner (C, C++ and Python suites).

    python3 test-suites/regression/python/run_regression.py            # everything
    python3 test-suites/regression/python/run_regression.py --fast     # skip mutation testing
    python3 test-suites/regression/python/run_regression.py --only native cli
    python3 test-suites/regression/python/run_regression.py --list

Groups
    native     builds and runs the C and C++ suites (strict flags, leak/heap-guard checked)
    ctypes     the C core driven from Python against Python oracles
    cli        the `sleela` compiler: language semantics, contract, transpilers
    corpus     hostile and malformed inputs (annotations, traversal, HTTP frames, parser, limits)
    integrity  SHA-256 gate tooling, committed manifest, /lib inventory
    mutation   injects defects into the code under test; every one must be caught

Exit status 0 only when every test passes.  KNOWN ISSUES (defects the suites
have found and that are still open) are listed at the end; they do not change the
exit status.  Environment: CC, CXX, SLEELA_BIN, REG_SANITIZE=1 (ASan/UBSan, when
the toolchain ships them), REG_SKIP_MUTATION=1.
"""
from __future__ import annotations

import argparse
import json
import os
import sys
import time
import unittest
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))

import regression_support as rs  # noqa: E402

GROUPS = {
    "native": ["test_native_suites"],
    "ctypes": ["test_ctypes_differential"],
    "cli": ["test_cli_language", "test_cli_contract", "test_cli_transpilers"],
    "corpus": ["test_corpus"],
    "integrity": ["test_integrity_tools", "test_doc_versions"],
    "mutation": ["test_mutation_kill"],
}


def describe_environment() -> None:
    import shutil
    import subprocess
    print("environment")
    print(f"  repository : {rs.ROOT}")
    for tool in (rs.CC, rs.CXX):
        found = shutil.which(tool)
        ver = ""
        if found:
            ver = subprocess.run([found, "--version"], capture_output=True, text=True).stdout.splitlines()[0]
        print(f"  {tool:<10} : {ver or 'NOT FOUND'}")
    binary = rs.sleela_binary()
    if binary:
        banner = subprocess.run([str(binary), "version"], capture_output=True, text=True, env=rs.cli_env()).stdout
        line = next((ln for ln in banner.splitlines() if "SLeeLa" in ln), "?")
        print(f"  sleela     : {binary}\n               {line}")
    else:
        print("  sleela     : NOT FOUND (CLI suites will be skipped; build impl/build/sleela)")
    print(f"  java/javac : {'yes' if shutil.which('javac') and shutil.which('java') else 'no (JVM differential skipped)'}")
    print(f"  sanitizers : {'ASan+UBSan requested' if os.environ.get('REG_SANITIZE') == '1' else 'off (REG_SANITIZE=1 to enable)'}")
    print()


def print_known_issues() -> None:
    if not rs.KNOWN_FILE.exists():
        return
    found = json.loads(rs.KNOWN_FILE.read_text())
    merged: dict = {}
    for e in found:
        cur = merged.setdefault(e["id"], {"open": False, "description": e["description"], "sources": set()})
        cur["open"] = cur["open"] or e["open"]
        cur["sources"].add(e["source"])
        if e["open"] and not e["description"].startswith("KNOWN-ISSUE"):
            cur["description"] = e["description"]
    if not merged:
        return
    print("\nKNOWN ISSUES (reported by the suites; they do not fail the run)")
    for ident in sorted(merged):
        e = merged[ident]
        state = "OPEN " if e["open"] else "FIXED"
        text = e["description"]
        if text.startswith("KNOWN-ISSUE"):
            text = text.split(":", 1)[-1].strip()
        print(f"  [{state}] {ident:<22} {text}")
        print(f"          found by: {', '.join(sorted(e['sources']))}")


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--only", nargs="+", choices=sorted(GROUPS), help="run only these groups")
    ap.add_argument("--fast", action="store_true", help="skip mutation testing")
    ap.add_argument("--sanitize", action="store_true", help="build with -fsanitize=address,undefined")
    ap.add_argument("--list", action="store_true", help="list tests and exit")
    ap.add_argument("-v", "--verbose", action="store_true")
    ap.add_argument("-k", dest="pattern", help="only tests whose id contains this text")
    ap.add_argument("--clean", action="store_true", help="delete build output and exit")
    args = ap.parse_args()

    if args.clean:
        import shutil
        shutil.rmtree(rs.BUILD, ignore_errors=True)
        return 0
    if args.fast:
        os.environ["REG_SKIP_MUTATION"] = "1"
    if args.sanitize:
        os.environ["REG_SANITIZE"] = "1"

    groups = args.only or [g for g in GROUPS if not (args.fast and g == "mutation")]
    loader = unittest.TestLoader()
    suite = unittest.TestSuite()
    for g in groups:
        for module in GROUPS[g]:
            suite.addTests(loader.loadTestsFromName(module))

    def flatten(s):
        for t in s:
            if isinstance(t, unittest.TestSuite):
                yield from flatten(t)
            else:
                yield t
    tests = [t for t in flatten(suite) if not args.pattern or args.pattern in t.id()]
    if args.list:
        for t in tests:
            print(t.id())
        print(f"{len(tests)} tests")
        return 0

    try:
        rs.sleela_binary()
    except rs.CompilerSelectionError as exc:
        print(f"run_regression: {exc}", file=sys.stderr)
        return 2

    rs.BUILD.mkdir(parents=True, exist_ok=True)
    if rs.KNOWN_FILE.exists():
        rs.KNOWN_FILE.unlink()
    describe_environment()
    sys.stdout.flush()

    selected = unittest.TestSuite(tests)
    started = time.time()
    result = unittest.TextTestRunner(verbosity=2 if args.verbose else 1, buffer=False).run(selected)
    elapsed = time.time() - started
    sys.stderr.flush()

    print_known_issues()
    rs.cleanup_scratch()
    ok = result.wasSuccessful()
    print(f"\nREGRESSION {'PASS' if ok else 'FAIL'}: {result.testsRun} tests, {len(result.failures)} failed, "
          f"{len(result.errors)} errors, {len(result.skipped)} skipped in {elapsed:.0f}s")
    for test, _ in result.failures + result.errors:
        print(f"  FAILED: {test.id()}")
    return 0 if ok else 1


if __name__ == "__main__":
    raise SystemExit(main())