test_integrity_tools.py
test-suites/regression/python/test_integrity_tools.py · 371 line(s) · py
"""Regression tests for the SHA-256 execution gate tooling and for repository
invariants that other tooling depends on.
tools/verify-before-execution.py fail-closed verifier (exit 0 pass, 1 fail, 2 unreadable)
tools/generate-sha256-manifest.py manifest generator
security/sha256-manifest.json the committed trust root
lib/LIBRARY.SYMBOLS.md / test-suites/test-library-inventory.sh /lib inventory
The verifier is the only thing standing between a tampered source file and
`sleela run`, so its rejection paths are exercised exhaustively.
"""
import hashlib
import json
import os
import re
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
import regression_support as rs
VERIFY = rs.ROOT / "tools" / "verify-before-execution.py"
GENERATE = rs.ROOT / "tools" / "generate-sha256-manifest.py"
COMMITTED = rs.ROOT / "security" / "sha256-manifest.json"
EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
def verify(manifest: Path, root: Path):
return subprocess.run([sys.executable, str(VERIFY), "--manifest", str(manifest), "--root", str(root)],
capture_output=True, text=True, timeout=300)
def sha(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
class VerifierRejectionPaths(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory(prefix="verify-", dir=str(rs.BUILD) if rs.BUILD.exists() else None)
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name) / "root"
(self.root / "sub").mkdir(parents=True)
(self.root / "a.txt").write_bytes(b"alpha\n")
(self.root / "sub" / "b.bin").write_bytes(bytes(range(256)) * 4)
(self.root / "empty").write_bytes(b"")
self.manifest = Path(self.tmp.name) / "m.json"
def write_manifest(self, files, **extra):
doc = {"algorithm": "sha256", "files": files}
doc.update(extra)
self.manifest.write_text(json.dumps(doc))
def good_entries(self):
return [{"path": "a.txt", "sha256": sha(b"alpha\n")},
{"path": "sub/b.bin", "sha256": sha(bytes(range(256)) * 4)},
{"path": "empty", "sha256": EMPTY_SHA256}]
def test_correct_manifest_passes(self):
self.write_manifest(self.good_entries())
r = verify(self.manifest, self.root)
self.assertEqual(r.returncode, 0, r.stderr)
self.assertIn("PASS (3 item(s))", r.stdout)
def test_hex_digest_case_is_irrelevant(self):
e = self.good_entries()
e[0]["sha256"] = e[0]["sha256"].upper()
self.write_manifest(e)
self.assertEqual(verify(self.manifest, self.root).returncode, 0)
def test_algorithm_defaults_to_sha256_when_omitted(self):
self.manifest.write_text(json.dumps({"files": self.good_entries()}))
self.assertEqual(verify(self.manifest, self.root).returncode, 0)
def test_a_one_byte_change_is_detected_and_named(self):
self.write_manifest(self.good_entries())
(self.root / "a.txt").write_bytes(b"alphb\n")
r = verify(self.manifest, self.root)
self.assertEqual(r.returncode, 1)
self.assertIn("SHA-256 mismatch: a.txt", r.stderr)
self.assertIn("expected: " + sha(b"alpha\n"), r.stderr)
self.assertIn("actual: " + sha(b"alphb\n"), r.stderr)
self.assertIn("FAILED (1 item(s))", r.stderr)
def test_appended_and_truncated_content_is_detected(self):
self.write_manifest(self.good_entries())
data = (self.root / "sub" / "b.bin").read_bytes()
for mutated in (data + b"\0", data[:-1], b"", data[:10] + bytes([data[10] ^ 1]) + data[11:]):
(self.root / "sub" / "b.bin").write_bytes(mutated)
self.assertEqual(verify(self.manifest, self.root).returncode, 1)
(self.root / "sub" / "b.bin").write_bytes(data)
self.assertEqual(verify(self.manifest, self.root).returncode, 0)
def test_every_failing_entry_is_counted(self):
self.write_manifest(self.good_entries())
(self.root / "a.txt").write_bytes(b"x")
(self.root / "empty").write_bytes(b"x")
r = verify(self.manifest, self.root)
self.assertEqual(r.returncode, 1)
self.assertIn("FAILED (2 item(s))", r.stderr)
def test_missing_file_fails(self):
self.write_manifest(self.good_entries() + [{"path": "gone.txt", "sha256": EMPTY_SHA256}])
r = verify(self.manifest, self.root)
self.assertEqual(r.returncode, 1)
self.assertIn("missing file: gone.txt", r.stderr)
def test_a_directory_is_not_a_file(self):
self.write_manifest([{"path": "sub", "sha256": EMPTY_SHA256}])
self.assertEqual(verify(self.manifest, self.root).returncode, 1)
def test_paths_that_escape_the_root_are_rejected(self):
outside = Path(self.tmp.name) / "outside.txt"
outside.write_bytes(b"secret")
(self.root / "link").symlink_to(outside)
for path in ("../outside.txt", "sub/../../outside.txt", str(outside), "link"):
self.write_manifest([{"path": path, "sha256": sha(b"secret")}]) # digest is correct: still refused
r = verify(self.manifest, self.root)
self.assertEqual(r.returncode, 1, path)
self.assertIn("rejected path outside root", r.stderr, path)
def test_a_symlink_that_stays_inside_the_root_is_followed(self):
(self.root / "alias").symlink_to(self.root / "a.txt")
self.write_manifest([{"path": "alias", "sha256": sha(b"alpha\n")}])
self.assertEqual(verify(self.manifest, self.root).returncode, 0)
def test_unreadable_or_invalid_manifests_exit_2(self):
r = verify(Path(self.tmp.name) / "absent.json", self.root)
self.assertEqual(r.returncode, 2)
self.manifest.write_text("{not json")
self.assertEqual(verify(self.manifest, self.root).returncode, 2)
self.write_manifest([])
self.assertEqual(verify(self.manifest, self.root).returncode, 2)
self.manifest.write_text(json.dumps({"algorithm": "sha256"}))
self.assertEqual(verify(self.manifest, self.root).returncode, 2)
self.write_manifest(self.good_entries(), algorithm="md5")
self.assertEqual(verify(self.manifest, self.root).returncode, 2)
self.manifest.write_text(json.dumps({"files": "a.txt"}))
self.assertEqual(verify(self.manifest, self.root).returncode, 2)
def test_malformed_entries_fail_rather_than_being_skipped(self):
for entry in ("a.txt", 5, None, {"path": "a.txt"}, {"sha256": EMPTY_SHA256}, {"path": 3, "sha256": EMPTY_SHA256},
{"path": "a.txt", "sha256": ""}):
self.write_manifest(self.good_entries() + [entry])
r = verify(self.manifest, self.root)
self.assertEqual(r.returncode, 1, repr(entry))
def test_large_files_are_hashed_completely(self):
big = (b"0123456789abcdef" * 1024) * 3 * 1024 + b"tail" # 48 MiB + 4 spans several read chunks
(self.root / "big.bin").write_bytes(big)
self.write_manifest([{"path": "big.bin", "sha256": sha(big)}])
self.assertEqual(verify(self.manifest, self.root).returncode, 0)
(self.root / "big.bin").write_bytes(big[:-1] + b"X")
self.assertEqual(verify(self.manifest, self.root).returncode, 1)
class ManifestGenerator(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.dir = rs.fresh_dir("gen")
cls.out = cls.dir / "m.json"
r = subprocess.run([sys.executable, str(GENERATE), "--root", str(rs.ROOT), "--output", str(cls.out)],
capture_output=True, text=True, timeout=300)
assert r.returncode == 0, r.stderr
cls.doc = json.loads(cls.out.read_text())
cls.paths = [f["path"] for f in cls.doc["files"]]
def test_format(self):
self.assertEqual(self.doc["algorithm"], "sha256")
self.assertGreater(len(self.paths), 1000)
self.assertEqual(len(self.paths), len(set(self.paths)), "duplicate entries")
for f in self.doc["files"]:
self.assertRegex(f["sha256"], r"^[0-9a-f]{64}$")
self.assertFalse(f["path"].startswith("/") or ".." in Path(f["path"]).parts, f["path"])
self.assertNotIn("\\", f["path"])
def test_generation_is_deterministic(self):
again = self.dir / "again.json"
subprocess.run([sys.executable, str(GENERATE), "--root", str(rs.ROOT), "--output", str(again)],
check=True, capture_output=True, timeout=300)
self.assertEqual(again.read_bytes(), self.out.read_bytes())
def test_stdout_mode_matches_file_mode(self):
r = subprocess.run([sys.executable, str(GENERATE), "--root", str(rs.ROOT), "--stdout"],
capture_output=True, text=True, timeout=300)
self.assertEqual(r.returncode, 0)
self.assertEqual(json.loads(r.stdout), self.doc)
def test_trusted_computing_base_is_covered(self):
must = ["impl/core/sleela_core.c", "impl/core/sleela_utf16.c", "impl/frontend/lexer.cpp",
"impl/frontend/compiler.cpp", "SHEET.sheet", "tools/sleela-native-drive.py"]
for m in must:
self.assertIn(m, self.paths)
self.assertTrue(any(p.startswith("lib/") and p.endswith(".sleela") for p in self.paths))
self.assertEqual([p for p in self.paths if p.startswith("bash/")],
["bash/pixel_terminal.cpp", "bash/pixel_terminal.hpp"], "the vendored bash tree is out; only its 2 SLeeLa files are in")
self.assertFalse(any(p.startswith((".git/", "impl/build/")) for p in self.paths))
# user programs are deliberately not covered
self.assertFalse(any(p.startswith(("examples/", "tutorial/")) for p in self.paths))
def test_every_entry_matches_its_file_and_the_gate_accepts_the_manifest(self):
r = verify(self.out, rs.ROOT)
self.assertEqual(r.returncode, 0, r.stderr[-800:])
self.assertIn(f"PASS ({len(self.paths)} item(s))", r.stdout)
def test_generator_on_a_miniature_tree(self):
root = self.dir / "mini"
for rel, data in {"impl/core/x.c": b"int x;\n", "impl/frontend/y.cpp": b"int y;\n", "lib/A/B.sleela": b"class B {}\n",
"SHEET.sheet": b"sheet\n", "bash/vendored.c": b"nope\n", "impl/core/build/gen.c": b"gen\n", "lib/A/.build/x.sleela": b"class X {}\n",
"impl/core/notes.txt": b"ignored\n"}.items():
p = root / rel
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(data)
out = self.dir / "mini.json"
r = subprocess.run([sys.executable, str(GENERATE), "--root", str(root), "--output", str(out)],
capture_output=True, text=True, timeout=60)
self.assertEqual(r.returncode, 0, r.stderr)
paths = {f["path"]: f["sha256"] for f in json.loads(out.read_text())["files"]}
self.assertEqual(paths["impl/core/x.c"], sha(b"int x;\n"))
self.assertIn("lib/A/B.sleela", paths)
self.assertIn("SHEET.sheet", paths)
self.assertNotIn("bash/vendored.c", paths)
self.assertNotIn("impl/core/build/gen.c", paths) # generated output is never trusted source
self.assertNotIn("lib/A/.build/x.sleela", paths)
self.assertNotIn("impl/core/notes.txt", paths)
self.assertEqual(verify(out, root).returncode, 0)
(root / "lib/A/B.sleela").write_bytes(b"class B { }\n") # edit after generation
self.assertEqual(verify(out, root).returncode, 1)
class CommittedTrustRoot(unittest.TestCase):
def test_committed_manifest_matches_the_committed_tree(self):
"""security/sha256-manifest.json must describe the tree it ships with.
If this fails, `impl/Makefile verify-security` and every prebuilt
`working-binaries/*` invocation refuse to run on a fresh clone.
Fix: python3 tools/generate-sha256-manifest.py --root . --output security/sha256-manifest.json"""
self.assertTrue(COMMITTED.is_file())
r = verify(COMMITTED, rs.ROOT)
mismatched = re.findall(r"SHA-256 mismatch: (\S+)", r.stderr)
missing = re.findall(r"missing file: (\S+)", r.stderr)
self.assertEqual(r.returncode, 0,
f"{len(mismatched)} file(s) differ from the committed manifest, {len(missing)} missing:\n "
+ "\n ".join((mismatched + missing)[:20]))
def test_committed_manifest_is_well_formed_and_covers_the_gate_itself(self):
doc = json.loads(COMMITTED.read_text())
self.assertEqual(doc["algorithm"], "sha256")
paths = [f["path"] for f in doc["files"]]
self.assertEqual(len(paths), len(set(paths)))
for must in ("impl/core/sleela_core.c", "impl/frontend/driver.cpp", "SHEET.sheet"):
self.assertIn(must, paths)
for f in doc["files"]:
self.assertRegex(f["sha256"], r"^[0-9a-f]{64}$")
def test_every_manifest_entry_exists(self):
doc = json.loads(COMMITTED.read_text())
missing = [f["path"] for f in doc["files"] if not (rs.ROOT / f["path"]).is_file()]
self.assertEqual(missing, [])
class LibraryInventory(unittest.TestCase):
"""/lib is counted four independent ways; they must agree."""
@classmethod
def setUpClass(cls):
lib = rs.ROOT / "lib"
packages, sources, facades = set(), 0, 0
for dirpath, _dirs, files in os.walk(lib):
for name in files:
if not name.endswith(".sleela"):
continue
rel = Path(dirpath, name).relative_to(lib)
packages.add(rel.parts[0])
if name == "SLPackage.sleela":
facades += 1
else:
sources += 1
cls.fs = {"packages": len(packages), "sources": sources, "facades": facades, "symbols": sources + facades}
def test_generator_counts_equal_the_filesystem(self):
r = subprocess.run([sys.executable, str(rs.ROOT / "tools" / "generate-library-symbols.py"), "--print-counts"],
capture_output=True, text=True, timeout=120)
self.assertEqual(r.returncode, 0)
got = dict((k, int(v)) for k, v in re.findall(r"(\w+)=(\d+)", r.stdout))
self.assertEqual(got, self.fs)
def test_hardcoded_expectations_in_the_inventory_script_are_current(self):
text = (rs.ROOT / "test-suites" / "test-library-inventory.sh").read_text()
want = {k.lower(): int(v) for k, v in re.findall(r"^EXPECTED_(\w+)=(\d+)", text, re.M)}
self.assertEqual(want, self.fs, "update EXPECTED_* in test-suites/test-library-inventory.sh")
def test_symbols_manifest_header_and_rows_match(self):
text = (rs.ROOT / "lib" / "LIBRARY.SYMBOLS.md").read_text()
rows = [ln.split("\t") for ln in text.splitlines() if ln.count("\t") == 3 and ln.split("\t")[2].startswith("lib/")]
self.assertEqual(len(rows), self.fs["symbols"])
self.assertEqual(sum(1 for r in rows if r[3] == "facade"), self.fs["facades"])
self.assertEqual(len({r[2] for r in rows}), len(rows), "duplicate paths in LIBRARY.SYMBOLS.md")
for r in rows[:: max(1, len(rows) // 400)]: # spot-check 400 paths exist
self.assertTrue((rs.ROOT / r[2]).is_file(), r[2])
self.assertIn(f"library-source-files: {self.fs['sources']}", text)
def test_index_headline_revision_states_the_current_counts(self):
text = (rs.ROOT / "lib" / "LIBRARY.INDEX.md").read_text()
first = next(ln for ln in text.splitlines() if ln.startswith("> Revision"))
numbers = [int(n) for n in re.findall(r"\*\*(\d+)\*\*", first)]
self.assertIn(self.fs["sources"], numbers, first[:200])
self.assertIn(self.fs["symbols"], numbers, first[:200])
class RepositoryToolingHygiene(unittest.TestCase):
def test_every_python_tool_compiles(self):
bad = []
for p in sorted((rs.ROOT / "tools").rglob("*.py")):
try:
compile(p.read_text(encoding="utf-8"), str(p), "exec")
except (SyntaxError, UnicodeDecodeError) as e:
bad.append(f"{p.relative_to(rs.ROOT)}: {e}")
self.assertEqual(bad, [])
class CompilerSelection(unittest.TestCase):
"""Regression (was HARNESS-RELATIVE-SLEELA-BIN): a relative $SLEELA_BIN was
returned as given, and the CLI suites run the compiler from scratch
directories, so nearly every CLI test failed with FileNotFoundError. A
missing $SLEELA_BIN silently fell back to a different compiler."""
def setUp(self):
self.saved = os.environ.get("SLEELA_BIN")
self.cwd = os.getcwd()
def tearDown(self):
os.chdir(self.cwd)
if self.saved is None:
os.environ.pop("SLEELA_BIN", None)
else:
os.environ["SLEELA_BIN"] = self.saved
def test_relative_path_is_resolved_to_an_absolute_one(self):
with tempfile.TemporaryDirectory() as tmp:
exe = Path(tmp) / "bin" / "fake-sleela"
exe.parent.mkdir()
exe.write_text("#!/bin/sh\nexit 0\n")
exe.chmod(0o755)
os.chdir(tmp)
os.environ["SLEELA_BIN"] = "bin/fake-sleela"
got = rs.sleela_binary()
self.assertTrue(got.is_absolute(), got)
self.assertEqual(got, exe.resolve())
def test_explicit_path_that_is_not_an_executable_is_an_error(self):
with tempfile.TemporaryDirectory() as tmp:
not_exec = Path(tmp) / "plain.txt"
not_exec.write_text("x")
not_exec.chmod(0o644)
for value in (str(Path(tmp) / "missing"), str(not_exec), tmp):
os.environ["SLEELA_BIN"] = value
with self.assertRaises(rs.CompilerSelectionError, msg=value):
rs.sleela_binary()
def test_runner_refuses_to_start_with_a_bad_sleela_bin(self):
env = dict(os.environ, SLEELA_BIN="/nonexistent/sleela")
r = subprocess.run([sys.executable, str(rs.HERE / "run_regression.py"), "--only", "integrity"],
capture_output=True, text=True, env=env, timeout=120)
self.assertEqual(r.returncode, 2, r.stdout + r.stderr)
self.assertIn("SLEELA_BIN", r.stderr)
if __name__ == "__main__":
unittest.main()