test_cli_contract.py
test-suites/regression/python/test_cli_contract.py · 336 line(s) · py
"""Command-line contract of the `sleela` compiler/runtime.
Covers: version/syntax gating (SL-META-0001 section 4.4), the syntax 3.1
fixtures, subcommand error handling, the fail-closed SHA-256 execution gate,
and the persistent artifact format (`compile`, `validate-artifact`, `run`).
Fixtures are the ones committed under impl/tests/; they are copied into scratch
directories because the compiler writes `docs/keyword-supporters/` next to the
files it checks.
"""
import os
import random
import shutil
import unittest
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import regression_support as rs
TESTS = rs.ROOT / "impl" / "tests"
WORKERS = max(2, min(8, os.cpu_count() or 2))
def parallel(fn, items):
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
return list(pool.map(fn, items))
class NeedsCompiler(unittest.TestCase):
@classmethod
def setUpClass(cls):
if rs.sleela_binary() is None:
raise unittest.SkipTest("no sleela binary")
def run_fixture(directory: Path, name: str, mode: str = "run"):
return rs.run_sleela([mode, name], cwd=directory)
class VersionGating(NeedsCompiler):
"""impl/tests/version/run_version_tests.sh, as a table with exit-code assertions."""
ACCEPTED = ["syntax_1_0", "syntax_1_2", "syntax_1_3", "syntax_1_4", "syntax_3_0", "syntax_3_1", "syntax_3_01",
"syntax_missing", "material_2_5", "memory_2_5", "wide_integers", "int_double_widening",
"determine_2_6", "pure_3_0", "gasoline_3_0"]
REJECTED = ["syntax_0_9", "too_new", "minor_ahead", "material_too_early", "memory_too_early",
"determine_too_early", "pure_too_early", "gasoline_too_early", "malformed"]
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.dir = rs.fresh_dir("version")
for f in (TESTS / "version").glob("*.sleela"):
shutil.copy(f, cls.dir / f.name)
def check_all(self, names):
results = parallel(lambda n: (n, run_fixture(self.dir, n + ".sleela")), names)
return dict(results)
def test_supported_editions_and_markers_are_accepted(self):
for name, cp in self.check_all(self.ACCEPTED).items():
self.assertEqual(cp.returncode, 0, f"{name} must be accepted:\n{rs.diagnostics(cp)}")
def test_out_of_range_and_premature_features_are_rejected_with_a_reason(self):
for name, cp in self.check_all(self.REJECTED).items():
self.assertNotEqual(cp.returncode, 0, f"{name} must be rejected")
self.assertTrue(rs.diagnostics(cp).strip(), f"{name} rejected without a diagnostic")
self.assertEqual(rs.program_output(cp), [], f"{name} must not execute any code")
def test_range_diagnostics_name_the_bounds(self):
low = rs.diagnostics(run_fixture(self.dir, "syntax_0_9.sleela"))
high = rs.diagnostics(run_fixture(self.dir, "minor_ahead.sleela"))
self.assertIn("1.0", low)
self.assertIn("3.1", high)
def test_version_command_states_the_supported_range(self):
cp = rs.run_sleela(["version"], cwd=self.dir)
self.assertEqual(cp.returncode, 0)
banner = cp.stdout
self.assertIn("SLeeLa", banner)
self.assertIn("supported source syntax 1.0", banner)
self.assertIn("3.1", banner)
def test_unknown_pragma_edition_does_not_change_the_result_of_check(self):
cp = run_fixture(self.dir, "syntax_1_3.sleela", "check")
self.assertEqual(cp.returncode, 0)
self.assertIn("syntax_1_3.sleela: ok", cp.stdout)
self.assertIn("1.3", cp.stdout)
class Syntax31Fixtures(NeedsCompiler):
"""impl/tests/syntax31: char literals, array initialisers, multi-dimensional
arrays, labelled statements, finally on abrupt completion, and the 3.0
gates that must reject them."""
REJECT = {
"gate_length_in_3_0": ("check", "the array length member requires #sleela 3.1"),
"gate_nd_in_3_0": ("check", "multi-dimensional array creation requires #sleela 3.1"),
"bad_nd_sized_after_unsized": ("check", "array dimension expression after an empty"),
"bad_nd_negative": ("run", "negative array size"),
"bad_length_assign": ("check", "the array length is read-only"),
"bad_length_incr": ("check", "the array length is read-only"),
"gate_char_in_3_0": ("check", "character literal requires #sleela 3.1"),
"gate_label_in_3_0": ("check", "labeled statement requires #sleela 3.1"),
"gate_arrayinit_in_3_0": ("check", "array initializer requires #sleela 3.1"),
"bad_label": ("check", "break to undefined label 'nowhere'"),
}
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.src = TESTS / "syntax31"
def fixture_dir(self, label):
d = rs.fresh_dir(label)
for f in self.src.glob("*.sleela"):
shutil.copy(f, d / f.name)
return d
def test_every_fixture_is_covered_by_this_table_or_has_expected_output(self):
names = {p.stem for p in self.src.glob("*.sleela")}
covered = set(self.REJECT) | {p.name[: -len(".expected.txt")] for p in self.src.glob("*.expected.txt")}
self.assertEqual(names - covered, set(), "new syntax31 fixture without an expectation")
def test_programs_produce_their_expected_output(self):
names = sorted(p.name[: -len(".expected.txt")] for p in self.src.glob("*.expected.txt"))
self.assertGreaterEqual(len(names), 5)
def one(name):
d = self.fixture_dir("s31-" + name)
cp = run_fixture(d, name + ".sleela")
return name, cp, (self.src / f"{name}.expected.txt").read_text().splitlines()
for name, cp, want in parallel(one, names):
self.assertEqual(cp.returncode, 0, f"{name}: {rs.diagnostics(cp)}")
self.assertEqual(rs.program_output(cp), want, name)
def test_gates_and_malformed_programs_are_rejected_with_the_documented_message(self):
def one(item):
name, (mode, message) = item
d = self.fixture_dir("s31g-" + name)
return name, message, run_fixture(d, name + ".sleela", mode)
for name, message, cp in parallel(one, sorted(self.REJECT.items())):
self.assertNotEqual(cp.returncode, 0, f"{name} must be rejected")
self.assertIn(message, cp.stdout + cp.stderr, name)
class CommandLineErrors(NeedsCompiler):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.dir = rs.fresh_dir("cli")
rs.write_program(cls.dir, "Ok.sleela", '#sleela 3.1\nclass Ok { void main() { print("hi"); } }')
rs.write_program(cls.dir, "Syn.sleela", '#sleela 3.1\nclass Syn { void main() { print("hi" } }')
rs.write_program(cls.dir, "Sem.sleela", "#sleela 3.1\nclass Sem { void main() { print(undefinedVar); } }")
rs.write_program(cls.dir, "NoMain.sleela", "#sleela 3.1\nclass NoMain { void f() { } }")
rs.write_program(cls.dir, "Side.sleela", '#sleela 3.1\nclass Side { void main() { print("SIDE-EFFECT"); } }')
def test_no_arguments_and_unknown_subcommand_are_usage_errors(self):
self.assertEqual(rs.run_sleela([], cwd=self.dir).returncode, 2)
self.assertEqual(rs.run_sleela(["frobnicate"], cwd=self.dir).returncode, 2)
def test_missing_input_file_is_reported(self):
for mode in ("run", "check"):
cp = rs.run_sleela([mode, "does-not-exist.sleela"], cwd=self.dir)
self.assertEqual(cp.returncode, 1)
self.assertIn("cannot open", cp.stderr)
def test_syntax_error_reports_position(self):
for mode in ("run", "check"):
cp = rs.run_sleela([mode, "Syn.sleela"], cwd=self.dir)
self.assertEqual(cp.returncode, 1)
self.assertRegex(rs.diagnostics(cp), r"Syn\.sleela: Syntax error \(line 2, col \d+\): expected '\)'")
self.assertEqual(rs.program_output(cp), [])
def test_semantic_errors_and_missing_main_are_rejected_before_running(self):
cp = rs.run_sleela(["run", "Sem.sleela"], cwd=self.dir)
self.assertEqual(cp.returncode, 1)
self.assertIn("use of undeclared variable 'undefinedVar'", rs.diagnostics(cp))
cp = rs.run_sleela(["run", "NoMain.sleela"], cwd=self.dir)
self.assertEqual(cp.returncode, 1)
self.assertIn("no 'main' method found", rs.diagnostics(cp))
def test_check_validates_without_executing(self):
cp = rs.run_sleela(["check", "Side.sleela"], cwd=self.dir)
self.assertEqual(cp.returncode, 0)
self.assertIn("Side.sleela: ok", cp.stdout)
self.assertNotIn("SIDE-EFFECT", cp.stdout)
self.assertEqual(rs.run_sleela(["check", "Sem.sleela"], cwd=self.dir).returncode, 1)
def test_execution_is_deterministic(self):
runs = [rs.program_output(rs.run_sleela(["run", "Side.sleela"], cwd=self.dir)) for _ in range(3)]
self.assertEqual(runs[0], ["SIDE-EFFECT"])
self.assertEqual(runs[0], runs[1])
self.assertEqual(runs[1], runs[2])
class ExecutionGate(NeedsCompiler):
"""The compiler must refuse to build, run or check anything unless the
trusted SHA-256 manifest verifies (fail closed)."""
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.dir = rs.fresh_dir("gate")
rs.write_program(cls.dir, "G.sleela", '#sleela 3.1\nclass G { void main() { print("RAN"); } }')
import json
good = json.loads(rs.fast_manifest().read_text())
bad = json.loads(json.dumps(good))
bad["files"][0]["sha256"] = "0" * 64
missing = json.loads(json.dumps(good))
missing["files"].append({"path": "no/such/file.txt", "sha256": "0" * 64})
outside = json.loads(json.dumps(good))
outside["files"].append({"path": "../../../../etc/passwd", "sha256": "0" * 64})
cls.manifests = {}
for label, doc in (("tampered", bad), ("missing-file", missing), ("outside-root", outside),
("empty", {"algorithm": "sha256", "files": []}),
("wrong-algorithm", {"algorithm": "md5", "files": good["files"]})):
p = cls.dir / f"{label}.json"
p.write_text(json.dumps(doc))
cls.manifests[label] = p
(cls.dir / "garbage.json").write_text("{not json")
cls.manifests["garbage"] = cls.dir / "garbage.json"
cls.manifests["absent"] = cls.dir / "absent.json"
def test_bad_manifests_block_run_check_and_compile(self):
for label, manifest in self.manifests.items():
env = rs.cli_env(manifest)
for args in (["run", "G.sleela"], ["check", "G.sleela"], ["compile", "G.sleela", "-o", "G.out"]):
cp = rs.run_sleela(args, cwd=self.dir, env=env)
self.assertNotEqual(cp.returncode, 0, f"{label}: {args[0]} must fail closed")
self.assertNotIn("RAN", cp.stdout, f"{label}: program must not run")
self.assertFalse((self.dir / "G.out").exists(), f"{label}: no artifact may be produced")
def test_tampered_manifest_names_the_offending_file(self):
cp = rs.run_sleela(["run", "G.sleela"], cwd=self.dir, env=rs.cli_env(self.manifests["tampered"]))
self.assertIn("SHA-256 mismatch: SHEET.sheet", cp.stdout + cp.stderr)
def test_valid_manifest_allows_execution(self):
cp = rs.run_sleela(["run", "G.sleela"], cwd=self.dir)
self.assertEqual(cp.returncode, 0)
self.assertEqual(rs.program_output(cp), ["RAN"])
def test_version_does_not_require_the_gate(self):
cp = rs.run_sleela(["version"], cwd=self.dir, env=rs.cli_env(self.manifests["tampered"]))
self.assertEqual(cp.returncode, 0)
class Artifacts(NeedsCompiler):
SOURCE = '#sleela 1.3\nclass Art { void main() { print("ART-OK"); } }\n'
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.dir = rs.fresh_dir("artifact")
rs.write_program(cls.dir, "art.sleela", cls.SOURCE)
cp = rs.run_sleela(["compile", "art.sleela", "-o", "art.out"], cwd=cls.dir)
assert cp.returncode == 0, cp.stderr
cls.blob = (cls.dir / "art.out").read_bytes()
def validate(self, data: bytes, name: str = "probe.out"):
p = self.dir / name
p.write_bytes(data)
return rs.run_sleela(["validate-artifact", name], cwd=self.dir)
def test_artifact_has_the_documented_header(self):
self.assertTrue(self.blob.startswith(b"SLEELA-ART\x01"))
self.assertEqual(self.blob[11:15], b"\x06\x00\x00\x00") # artifact format 6
self.assertEqual(self.blob[15:19], b"\x04\x03\x02\x01") # endianness marker
self.assertIn(b"ART-OK", self.blob)
def test_valid_artifact_validates_and_runs_like_the_source(self):
cp = rs.run_sleela(["validate-artifact", "art.out"], cwd=self.dir)
self.assertEqual(cp.returncode, 0)
self.assertIn("artifact ABI ok", cp.stdout)
self.assertIn("format 6", cp.stdout)
from_artifact = rs.program_output(rs.run_sleela(["run", "art.out"], cwd=self.dir))
from_source = rs.program_output(rs.run_sleela(["run", "art.sleela"], cwd=self.dir))
self.assertEqual(from_artifact, ["ART-OK"])
self.assertEqual(from_artifact, from_source)
def test_compilation_is_deterministic(self):
again = rs.run_sleela(["compile", "art.sleela", "-o", "art2.out"], cwd=self.dir)
self.assertEqual(again.returncode, 0)
self.assertEqual((self.dir / "art2.out").read_bytes(), self.blob)
def test_every_truncation_is_rejected(self):
def one(n):
return n, self.validate(self.blob[:n], f"trunc-{n}.out").returncode
bad = [n for n, rc in parallel(one, range(len(self.blob))) if rc == 0]
self.assertEqual(bad, [], f"truncated artifacts accepted at lengths {bad[:10]}")
def test_every_single_byte_corruption_of_the_header_is_rejected(self):
header = 53 # magic, version, format, endianness, section table (see the header test above)
def one(i):
m = bytearray(self.blob); m[i] ^= 0xFF
return i, self.validate(bytes(m), f"hdr-{i}.out").returncode
accepted = [i for i, rc in parallel(one, range(header)) if rc == 0]
self.assertEqual(accepted, [], f"corrupted header bytes accepted: {accepted}")
def test_arbitrary_corruption_never_crashes_the_validator(self):
rng = random.Random(0xA27)
cases = []
for _ in range(300):
m = bytearray(self.blob)
for _ in range(rng.randint(1, 4)):
m[rng.randrange(len(m))] = rng.randrange(256)
cases.append(bytes(m))
cases += [b"", b"\x00", b"SLEELA-ART", b"\xff" * 300, self.blob * 2, self.blob + b"\x00" * 64]
def one(item):
idx, data = item
return self.validate(data, f"fz-{idx}.out").returncode
codes = parallel(one, list(enumerate(cases)))
self.assertTrue(all(c in (0, 1) for c in codes), f"validator crashed or hung: {sorted(set(codes))}")
def test_corrupt_artifact_does_not_execute(self):
m = bytearray(self.blob)
m[0] ^= 0xFF
(self.dir / "bad.out").write_bytes(bytes(m))
cp = rs.run_sleela(["run", "bad.out"], cwd=self.dir)
self.assertNotEqual(cp.returncode, 0)
self.assertNotIn("ART-OK", cp.stdout)
def test_compile_failure_leaves_no_artifact(self):
rs.write_program(self.dir, "broken.sleela", "#sleela 3.1\nclass B { void main() { print( } }")
cp = rs.run_sleela(["compile", "broken.sleela", "-o", "broken.out"], cwd=self.dir)
self.assertNotEqual(cp.returncode, 0)
self.assertFalse((self.dir / "broken.out").exists())
def test_trailing_bytes_are_ignored(self):
"""CHARACTERIZATION: validate-artifact checks the declared structure and
does not reject bytes appended after it."""
self.assertEqual(self.validate(self.blob + b"\x00").returncode, 0)
if __name__ == "__main__":
unittest.main()