run_sleela_array_string_vectors.py
comparison/tests/run_sleela_array_string_vectors.py · 100 line(s) · py
#!/usr/bin/env python3
"""Run the SCS-TEST-003 array/string subset supported by the SLeeLa VM."""
from __future__ import annotations
import json
import os
import pathlib
import subprocess
import sys
ROOT = pathlib.Path(__file__).resolve().parents[2]
BINARY = pathlib.Path(os.environ.get("SLEELA_BIN", ROOT / "impl/build/sleela"))
CASES = [
("valid", "comparison/tests/sleela/array_string_valid.sleela", 0),
("negative-index", "comparison/tests/sleela/array_negative_index.sleela", 1),
("one-past-end", "comparison/tests/sleela/array_past_end.sleela", 1),
]
EXPECTED_VALID = [
"ARRAY-EMPTY=0",
"ARRAY-SINGLE=1",
"ARRAY-FIRST=11",
"ARRAY-LAST=33",
"STRING-LITERAL-EQUALITY=PASS",
"STRING-FIXTURE=USEFFUL",
]
# The runtime emits documented security and operational telemetry on stdout
# before the program's own output. Strip only those known diagnostic records.
DIAGNOSTIC_PREFIXES = (
"[memory-manager]",
"[security]",
"[defender]",
"verification: PASS (",
)
RESULT = {"suite": "SCS-TEST-003-sleela-vm", "cases": {}, "status": "FAIL"}
def program_lines(output: str) -> list[str]:
lines = output.replace("\r\n", "\n").splitlines()
return [
line for line in lines
if not line.startswith(DIAGNOSTIC_PREFIXES)
]
def main() -> int:
if not BINARY.is_file():
raise FileNotFoundError(f"SLeeLa executable not found: {BINARY}")
env = os.environ.copy()
results = {}
for name, relative, expected_code in CASES:
command = [str(BINARY), "run", str(ROOT / relative)]
completed = subprocess.run(
command, cwd=ROOT, env=env, text=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
)
raw_output = completed.stdout.replace("\r\n", "\n").splitlines()
actual_output = program_lines(completed.stdout)
results[name] = {
"command": command,
"exit_code": completed.returncode,
"output": raw_output,
"program_output": actual_output,
}
if name == "valid":
while actual_output and not actual_output[-1].strip():
actual_output.pop()
if completed.returncode != expected_code or actual_output != EXPECTED_VALID:
raise AssertionError(
f"valid VM vectors failed: exit={completed.returncode}, "
f"expected={EXPECTED_VALID!r}, actual={actual_output!r}"
)
print("PASS SLeeLa VM arrays and USEFFUL string-literal checks: 6 assertions")
else:
if completed.returncode != expected_code or "array index out of range" not in completed.stdout:
raise AssertionError(
f"{name} was not rejected as expected; exit={completed.returncode}, "
f"output={completed.stdout!r}"
)
print(f"PASS SLeeLa VM {name} rejection")
RESULT["cases"] = results
RESULT["status"] = "PASS"
RESULT["platform"] = sys.platform
RESULT["python"] = sys.version.split()[0]
(ROOT / "comparison-sleela-array-string-results.json").write_text(
json.dumps(RESULT, indent=2, ensure_ascii=True) + "\n", encoding="utf-8"
)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except Exception as exc:
RESULT["error"] = str(exc)
(ROOT / "comparison-sleela-array-string-results.json").write_text(
json.dumps(RESULT, indent=2, ensure_ascii=True) + "\n", encoding="utf-8"
)
print(f"FAIL SLeeLa VM array/string vectors: {exc}", file=sys.stderr)
raise