Home /Modules/comparison/run_all_vm_matrix.py

run_all_vm_matrix.py

comparison/tests/run_all_vm_matrix.py · 78 line(s) · py

#!/usr/bin/env python3
"""Run the Basic Native VM and all eleven runnable Premium SLVMs."""
from __future__ import annotations
import json, pathlib, subprocess, sys

ROOT = pathlib.Path(__file__).resolve().parents[2]
CLI = ROOT / "impl" / "build" / ("sleela.exe" if sys.platform == "win32" else "sleela")
SOURCE = ROOT / "comparison" / "tests" / "sleela" / "common_vectors.sleela"
EXPECTED = ["42", "15", "hello"]
RESULT = {"suite": "SCS-VM-MATRIX-001", "status": "PASS",
          "expected_output": EXPECTED, "standard_vms": [], "runnable_vms": []}

def is_runtime_diagnostic(line: str) -> bool:
    """Recognize known runtime/security diagnostics without masking program output."""
    return line.startswith((
        "[memory-manager]",
        "[security]",
        "[defender]",
        "verification:",
        "sleelvac:",
        "Running ",
    ))

def run(label, command):
    p = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT, check=True, timeout=180)
    lines = [x.strip() for x in p.stdout.replace("\\r\\n", "\\n").splitlines() if x.strip()]
    diagnostics = [line for line in lines if is_runtime_diagnostic(line)]
    actual = [line for line in lines if not is_runtime_diagnostic(line)]
    if actual != EXPECTED:
        raise AssertionError(
            f"{label}: expected program output {EXPECTED!r}, got {actual!r}\\n"
            f"recognized diagnostics ({len(diagnostics)}):\\n"
            f"{chr(10).join(diagnostics)}\\nfull output:\\n{p.stdout}"
        )
    RESULT["runnable_vms"].append({
        "target": label, "status": "PASS", "output": actual,
        "runtime_diagnostics_count": len(diagnostics),
        "runtime_diagnostics": diagnostics,
    })
    print(
        f"PASS {label}: {' | '.join(actual)} "
        f"(recognized runtime diagnostics: {len(diagnostics)})",
        flush=True,
    )

def main():
    if not CLI.is_file() or not SOURCE.is_file():
        raise FileNotFoundError("Native CLI or shared workload is missing")
    for n in range(1, 11):
        folder = ROOT / "virtual-machine" / "vm-version" / f"{n:03d}"
        manifest = folder / "MODIUM.json"
        source = folder / f"SLVmVersion{n:03d}.sleela"
        if not manifest.is_file() or not source.is_file():
            raise FileNotFoundError(f"Standard VM {n:03d} package is incomplete")
        RESULT["standard_vms"].append({"generation": n, "identity": f"idea-vm-{n:03d}",
            "status": "REGISTERED_NOT_BOOTABLE"})
    print("PASS Standard VM inventory: 10/10 registered; execution gated", flush=True)
    run("Basic Native VM", [str(CLI), "run", str(SOURCE)])
    for n in range(1, 12):
        run(f"Premium SLVM/{n}", [str(CLI), "vm", "run", "premium", str(n), str(SOURCE)])
    RESULT["counts"] = {"standard_registered_not_bootable": 10, "runnable_executed": 12,
                        "premium_executed": 11, "basic_native_executed": 1}
    (ROOT / "comparison-vm-matrix-results.json").write_text(
        json.dumps(RESULT, indent=2) + "\n", encoding="utf-8")
    print("PASS VM matrix: 12/12 runnable targets; 10 Standard entries inventoried")

if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except Exception as exc:
        RESULT["status"] = "FAIL"
        RESULT["error"] = str(exc)
        (ROOT / "comparison-vm-matrix-results.json").write_text(
            json.dumps(RESULT, indent=2) + "\n", encoding="utf-8")
        print(f"FAIL VM matrix: {exc}", file=sys.stderr)
        raise