Home /Modules/comparison/run_sleela_array_string_vectors.py

run_sleela_array_string_vectors.py

comparison/tests/run_sleela_array_string_vectors.py · 100 line(s) · py

#!/usr/bin/env python3
"""Run the SCS-TEST-003 array/string subset supported by the SLeeLa VM."""
from __future__ import annotations

import json
import os
import pathlib
import subprocess
import sys

ROOT = pathlib.Path(__file__).resolve().parents[2]
BINARY = pathlib.Path(os.environ.get("SLEELA_BIN", ROOT / "impl/build/sleela"))
CASES = [
    ("valid", "comparison/tests/sleela/array_string_valid.sleela", 0),
    ("negative-index", "comparison/tests/sleela/array_negative_index.sleela", 1),
    ("one-past-end", "comparison/tests/sleela/array_past_end.sleela", 1),
]
EXPECTED_VALID = [
    "ARRAY-EMPTY=0",
    "ARRAY-SINGLE=1",
    "ARRAY-FIRST=11",
    "ARRAY-LAST=33",
    "STRING-LITERAL-EQUALITY=PASS",
    "STRING-FIXTURE=USEFFUL",
]
# The runtime emits documented security and operational telemetry on stdout
# before the program's own output. Strip only those known diagnostic records.
DIAGNOSTIC_PREFIXES = (
    "[memory-manager]",
    "[security]",
    "[defender]",
    "verification: PASS (",
)
RESULT = {"suite": "SCS-TEST-003-sleela-vm", "cases": {}, "status": "FAIL"}


def program_lines(output: str) -> list[str]:
    lines = output.replace("\r\n", "\n").splitlines()
    return [
        line for line in lines
        if not line.startswith(DIAGNOSTIC_PREFIXES)
    ]


def main() -> int:
    if not BINARY.is_file():
        raise FileNotFoundError(f"SLeeLa executable not found: {BINARY}")
    env = os.environ.copy()
    results = {}
    for name, relative, expected_code in CASES:
        command = [str(BINARY), "run", str(ROOT / relative)]
        completed = subprocess.run(
            command, cwd=ROOT, env=env, text=True,
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
        )
        raw_output = completed.stdout.replace("\r\n", "\n").splitlines()
        actual_output = program_lines(completed.stdout)
        results[name] = {
            "command": command,
            "exit_code": completed.returncode,
            "output": raw_output,
            "program_output": actual_output,
        }
        if name == "valid":
            while actual_output and not actual_output[-1].strip():
                actual_output.pop()
            if completed.returncode != expected_code or actual_output != EXPECTED_VALID:
                raise AssertionError(
                    f"valid VM vectors failed: exit={completed.returncode}, "
                    f"expected={EXPECTED_VALID!r}, actual={actual_output!r}"
                )
            print("PASS SLeeLa VM arrays and USEFFUL string-literal checks: 6 assertions")
        else:
            if completed.returncode != expected_code or "array index out of range" not in completed.stdout:
                raise AssertionError(
                    f"{name} was not rejected as expected; exit={completed.returncode}, "
                    f"output={completed.stdout!r}"
                )
            print(f"PASS SLeeLa VM {name} rejection")
    RESULT["cases"] = results
    RESULT["status"] = "PASS"
    RESULT["platform"] = sys.platform
    RESULT["python"] = sys.version.split()[0]
    (ROOT / "comparison-sleela-array-string-results.json").write_text(
        json.dumps(RESULT, indent=2, ensure_ascii=True) + "\n", encoding="utf-8"
    )
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except Exception as exc:
        RESULT["error"] = str(exc)
        (ROOT / "comparison-sleela-array-string-results.json").write_text(
            json.dumps(RESULT, indent=2, ensure_ascii=True) + "\n", encoding="utf-8"
        )
        print(f"FAIL SLeeLa VM array/string vectors: {exc}", file=sys.stderr)
        raise