Home /Modules/test-suites/test_integrity_tools.py

test_integrity_tools.py

test-suites/regression/python/test_integrity_tools.py · 371 line(s) · py

"""Regression tests for the SHA-256 execution gate tooling and for repository
invariants that other tooling depends on.

  tools/verify-before-execution.py   fail-closed verifier (exit 0 pass, 1 fail, 2 unreadable)
  tools/generate-sha256-manifest.py  manifest generator
  security/sha256-manifest.json      the committed trust root
  lib/LIBRARY.SYMBOLS.md / test-suites/test-library-inventory.sh   /lib inventory

The verifier is the only thing standing between a tampered source file and
`sleela run`, so its rejection paths are exercised exhaustively.
"""
import hashlib
import json
import os
import re
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

import regression_support as rs

VERIFY = rs.ROOT / "tools" / "verify-before-execution.py"
GENERATE = rs.ROOT / "tools" / "generate-sha256-manifest.py"
COMMITTED = rs.ROOT / "security" / "sha256-manifest.json"
EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"


def verify(manifest: Path, root: Path):
    return subprocess.run([sys.executable, str(VERIFY), "--manifest", str(manifest), "--root", str(root)],
                          capture_output=True, text=True, timeout=300)


def sha(data: bytes) -> str:
    return hashlib.sha256(data).hexdigest()


class VerifierRejectionPaths(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.TemporaryDirectory(prefix="verify-", dir=str(rs.BUILD) if rs.BUILD.exists() else None)
        self.addCleanup(self.tmp.cleanup)
        self.root = Path(self.tmp.name) / "root"
        (self.root / "sub").mkdir(parents=True)
        (self.root / "a.txt").write_bytes(b"alpha\n")
        (self.root / "sub" / "b.bin").write_bytes(bytes(range(256)) * 4)
        (self.root / "empty").write_bytes(b"")
        self.manifest = Path(self.tmp.name) / "m.json"

    def write_manifest(self, files, **extra):
        doc = {"algorithm": "sha256", "files": files}
        doc.update(extra)
        self.manifest.write_text(json.dumps(doc))

    def good_entries(self):
        return [{"path": "a.txt", "sha256": sha(b"alpha\n")},
                {"path": "sub/b.bin", "sha256": sha(bytes(range(256)) * 4)},
                {"path": "empty", "sha256": EMPTY_SHA256}]

    def test_correct_manifest_passes(self):
        self.write_manifest(self.good_entries())
        r = verify(self.manifest, self.root)
        self.assertEqual(r.returncode, 0, r.stderr)
        self.assertIn("PASS (3 item(s))", r.stdout)

    def test_hex_digest_case_is_irrelevant(self):
        e = self.good_entries()
        e[0]["sha256"] = e[0]["sha256"].upper()
        self.write_manifest(e)
        self.assertEqual(verify(self.manifest, self.root).returncode, 0)

    def test_algorithm_defaults_to_sha256_when_omitted(self):
        self.manifest.write_text(json.dumps({"files": self.good_entries()}))
        self.assertEqual(verify(self.manifest, self.root).returncode, 0)

    def test_a_one_byte_change_is_detected_and_named(self):
        self.write_manifest(self.good_entries())
        (self.root / "a.txt").write_bytes(b"alphb\n")
        r = verify(self.manifest, self.root)
        self.assertEqual(r.returncode, 1)
        self.assertIn("SHA-256 mismatch: a.txt", r.stderr)
        self.assertIn("expected: " + sha(b"alpha\n"), r.stderr)
        self.assertIn("actual:   " + sha(b"alphb\n"), r.stderr)
        self.assertIn("FAILED (1 item(s))", r.stderr)

    def test_appended_and_truncated_content_is_detected(self):
        self.write_manifest(self.good_entries())
        data = (self.root / "sub" / "b.bin").read_bytes()
        for mutated in (data + b"\0", data[:-1], b"", data[:10] + bytes([data[10] ^ 1]) + data[11:]):
            (self.root / "sub" / "b.bin").write_bytes(mutated)
            self.assertEqual(verify(self.manifest, self.root).returncode, 1)
        (self.root / "sub" / "b.bin").write_bytes(data)
        self.assertEqual(verify(self.manifest, self.root).returncode, 0)

    def test_every_failing_entry_is_counted(self):
        self.write_manifest(self.good_entries())
        (self.root / "a.txt").write_bytes(b"x")
        (self.root / "empty").write_bytes(b"x")
        r = verify(self.manifest, self.root)
        self.assertEqual(r.returncode, 1)
        self.assertIn("FAILED (2 item(s))", r.stderr)

    def test_missing_file_fails(self):
        self.write_manifest(self.good_entries() + [{"path": "gone.txt", "sha256": EMPTY_SHA256}])
        r = verify(self.manifest, self.root)
        self.assertEqual(r.returncode, 1)
        self.assertIn("missing file: gone.txt", r.stderr)

    def test_a_directory_is_not_a_file(self):
        self.write_manifest([{"path": "sub", "sha256": EMPTY_SHA256}])
        self.assertEqual(verify(self.manifest, self.root).returncode, 1)

    def test_paths_that_escape_the_root_are_rejected(self):
        outside = Path(self.tmp.name) / "outside.txt"
        outside.write_bytes(b"secret")
        (self.root / "link").symlink_to(outside)
        for path in ("../outside.txt", "sub/../../outside.txt", str(outside), "link"):
            self.write_manifest([{"path": path, "sha256": sha(b"secret")}])      # digest is correct: still refused
            r = verify(self.manifest, self.root)
            self.assertEqual(r.returncode, 1, path)
            self.assertIn("rejected path outside root", r.stderr, path)

    def test_a_symlink_that_stays_inside_the_root_is_followed(self):
        (self.root / "alias").symlink_to(self.root / "a.txt")
        self.write_manifest([{"path": "alias", "sha256": sha(b"alpha\n")}])
        self.assertEqual(verify(self.manifest, self.root).returncode, 0)

    def test_unreadable_or_invalid_manifests_exit_2(self):
        r = verify(Path(self.tmp.name) / "absent.json", self.root)
        self.assertEqual(r.returncode, 2)
        self.manifest.write_text("{not json")
        self.assertEqual(verify(self.manifest, self.root).returncode, 2)
        self.write_manifest([])
        self.assertEqual(verify(self.manifest, self.root).returncode, 2)
        self.manifest.write_text(json.dumps({"algorithm": "sha256"}))
        self.assertEqual(verify(self.manifest, self.root).returncode, 2)
        self.write_manifest(self.good_entries(), algorithm="md5")
        self.assertEqual(verify(self.manifest, self.root).returncode, 2)
        self.manifest.write_text(json.dumps({"files": "a.txt"}))
        self.assertEqual(verify(self.manifest, self.root).returncode, 2)

    def test_malformed_entries_fail_rather_than_being_skipped(self):
        for entry in ("a.txt", 5, None, {"path": "a.txt"}, {"sha256": EMPTY_SHA256}, {"path": 3, "sha256": EMPTY_SHA256},
                      {"path": "a.txt", "sha256": ""}):
            self.write_manifest(self.good_entries() + [entry])
            r = verify(self.manifest, self.root)
            self.assertEqual(r.returncode, 1, repr(entry))

    def test_large_files_are_hashed_completely(self):
        big = (b"0123456789abcdef" * 1024) * 3 * 1024 + b"tail"                  # 48 MiB + 4 spans several read chunks
        (self.root / "big.bin").write_bytes(big)
        self.write_manifest([{"path": "big.bin", "sha256": sha(big)}])
        self.assertEqual(verify(self.manifest, self.root).returncode, 0)
        (self.root / "big.bin").write_bytes(big[:-1] + b"X")
        self.assertEqual(verify(self.manifest, self.root).returncode, 1)


class ManifestGenerator(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.dir = rs.fresh_dir("gen")
        cls.out = cls.dir / "m.json"
        r = subprocess.run([sys.executable, str(GENERATE), "--root", str(rs.ROOT), "--output", str(cls.out)],
                           capture_output=True, text=True, timeout=300)
        assert r.returncode == 0, r.stderr
        cls.doc = json.loads(cls.out.read_text())
        cls.paths = [f["path"] for f in cls.doc["files"]]

    def test_format(self):
        self.assertEqual(self.doc["algorithm"], "sha256")
        self.assertGreater(len(self.paths), 1000)
        self.assertEqual(len(self.paths), len(set(self.paths)), "duplicate entries")
        for f in self.doc["files"]:
            self.assertRegex(f["sha256"], r"^[0-9a-f]{64}$")
            self.assertFalse(f["path"].startswith("/") or ".." in Path(f["path"]).parts, f["path"])
            self.assertNotIn("\\", f["path"])

    def test_generation_is_deterministic(self):
        again = self.dir / "again.json"
        subprocess.run([sys.executable, str(GENERATE), "--root", str(rs.ROOT), "--output", str(again)],
                       check=True, capture_output=True, timeout=300)
        self.assertEqual(again.read_bytes(), self.out.read_bytes())

    def test_stdout_mode_matches_file_mode(self):
        r = subprocess.run([sys.executable, str(GENERATE), "--root", str(rs.ROOT), "--stdout"],
                           capture_output=True, text=True, timeout=300)
        self.assertEqual(r.returncode, 0)
        self.assertEqual(json.loads(r.stdout), self.doc)

    def test_trusted_computing_base_is_covered(self):
        must = ["impl/core/sleela_core.c", "impl/core/sleela_utf16.c", "impl/frontend/lexer.cpp",
                "impl/frontend/compiler.cpp", "SHEET.sheet", "tools/sleela-native-drive.py"]
        for m in must:
            self.assertIn(m, self.paths)
        self.assertTrue(any(p.startswith("lib/") and p.endswith(".sleela") for p in self.paths))
        self.assertEqual([p for p in self.paths if p.startswith("bash/")],
                         ["bash/pixel_terminal.cpp", "bash/pixel_terminal.hpp"], "the vendored bash tree is out; only its 2 SLeeLa files are in")
        self.assertFalse(any(p.startswith((".git/", "impl/build/")) for p in self.paths))
        # user programs are deliberately not covered
        self.assertFalse(any(p.startswith(("examples/", "tutorial/")) for p in self.paths))

    def test_every_entry_matches_its_file_and_the_gate_accepts_the_manifest(self):
        r = verify(self.out, rs.ROOT)
        self.assertEqual(r.returncode, 0, r.stderr[-800:])
        self.assertIn(f"PASS ({len(self.paths)} item(s))", r.stdout)

    def test_generator_on_a_miniature_tree(self):
        root = self.dir / "mini"
        for rel, data in {"impl/core/x.c": b"int x;\n", "impl/frontend/y.cpp": b"int y;\n", "lib/A/B.sleela": b"class B {}\n",
                          "SHEET.sheet": b"sheet\n", "bash/vendored.c": b"nope\n", "impl/core/build/gen.c": b"gen\n", "lib/A/.build/x.sleela": b"class X {}\n",
                          "impl/core/notes.txt": b"ignored\n"}.items():
            p = root / rel
            p.parent.mkdir(parents=True, exist_ok=True)
            p.write_bytes(data)
        out = self.dir / "mini.json"
        r = subprocess.run([sys.executable, str(GENERATE), "--root", str(root), "--output", str(out)],
                           capture_output=True, text=True, timeout=60)
        self.assertEqual(r.returncode, 0, r.stderr)
        paths = {f["path"]: f["sha256"] for f in json.loads(out.read_text())["files"]}
        self.assertEqual(paths["impl/core/x.c"], sha(b"int x;\n"))
        self.assertIn("lib/A/B.sleela", paths)
        self.assertIn("SHEET.sheet", paths)
        self.assertNotIn("bash/vendored.c", paths)
        self.assertNotIn("impl/core/build/gen.c", paths)          # generated output is never trusted source
        self.assertNotIn("lib/A/.build/x.sleela", paths)
        self.assertNotIn("impl/core/notes.txt", paths)
        self.assertEqual(verify(out, root).returncode, 0)
        (root / "lib/A/B.sleela").write_bytes(b"class B { }\n")            # edit after generation
        self.assertEqual(verify(out, root).returncode, 1)


class CommittedTrustRoot(unittest.TestCase):
    def test_committed_manifest_matches_the_committed_tree(self):
        """security/sha256-manifest.json must describe the tree it ships with.
        If this fails, `impl/Makefile verify-security` and every prebuilt
        `working-binaries/*` invocation refuse to run on a fresh clone.
        Fix: python3 tools/generate-sha256-manifest.py --root . --output security/sha256-manifest.json"""
        self.assertTrue(COMMITTED.is_file())
        r = verify(COMMITTED, rs.ROOT)
        mismatched = re.findall(r"SHA-256 mismatch: (\S+)", r.stderr)
        missing = re.findall(r"missing file: (\S+)", r.stderr)
        self.assertEqual(r.returncode, 0,
                         f"{len(mismatched)} file(s) differ from the committed manifest, {len(missing)} missing:\n  "
                         + "\n  ".join((mismatched + missing)[:20]))

    def test_committed_manifest_is_well_formed_and_covers_the_gate_itself(self):
        doc = json.loads(COMMITTED.read_text())
        self.assertEqual(doc["algorithm"], "sha256")
        paths = [f["path"] for f in doc["files"]]
        self.assertEqual(len(paths), len(set(paths)))
        for must in ("impl/core/sleela_core.c", "impl/frontend/driver.cpp", "SHEET.sheet"):
            self.assertIn(must, paths)
        for f in doc["files"]:
            self.assertRegex(f["sha256"], r"^[0-9a-f]{64}$")

    def test_every_manifest_entry_exists(self):
        doc = json.loads(COMMITTED.read_text())
        missing = [f["path"] for f in doc["files"] if not (rs.ROOT / f["path"]).is_file()]
        self.assertEqual(missing, [])


class LibraryInventory(unittest.TestCase):
    """/lib is counted four independent ways; they must agree."""

    @classmethod
    def setUpClass(cls):
        lib = rs.ROOT / "lib"
        packages, sources, facades = set(), 0, 0
        for dirpath, _dirs, files in os.walk(lib):
            for name in files:
                if not name.endswith(".sleela"):
                    continue
                rel = Path(dirpath, name).relative_to(lib)
                packages.add(rel.parts[0])
                if name == "SLPackage.sleela":
                    facades += 1
                else:
                    sources += 1
        cls.fs = {"packages": len(packages), "sources": sources, "facades": facades, "symbols": sources + facades}

    def test_generator_counts_equal_the_filesystem(self):
        r = subprocess.run([sys.executable, str(rs.ROOT / "tools" / "generate-library-symbols.py"), "--print-counts"],
                           capture_output=True, text=True, timeout=120)
        self.assertEqual(r.returncode, 0)
        got = dict((k, int(v)) for k, v in re.findall(r"(\w+)=(\d+)", r.stdout))
        self.assertEqual(got, self.fs)

    def test_hardcoded_expectations_in_the_inventory_script_are_current(self):
        text = (rs.ROOT / "test-suites" / "test-library-inventory.sh").read_text()
        want = {k.lower(): int(v) for k, v in re.findall(r"^EXPECTED_(\w+)=(\d+)", text, re.M)}
        self.assertEqual(want, self.fs, "update EXPECTED_* in test-suites/test-library-inventory.sh")

    def test_symbols_manifest_header_and_rows_match(self):
        text = (rs.ROOT / "lib" / "LIBRARY.SYMBOLS.md").read_text()
        rows = [ln.split("\t") for ln in text.splitlines() if ln.count("\t") == 3 and ln.split("\t")[2].startswith("lib/")]
        self.assertEqual(len(rows), self.fs["symbols"])
        self.assertEqual(sum(1 for r in rows if r[3] == "facade"), self.fs["facades"])
        self.assertEqual(len({r[2] for r in rows}), len(rows), "duplicate paths in LIBRARY.SYMBOLS.md")
        for r in rows[:: max(1, len(rows) // 400)]:                                  # spot-check 400 paths exist
            self.assertTrue((rs.ROOT / r[2]).is_file(), r[2])
        self.assertIn(f"library-source-files: {self.fs['sources']}", text)

    def test_index_headline_revision_states_the_current_counts(self):
        text = (rs.ROOT / "lib" / "LIBRARY.INDEX.md").read_text()
        first = next(ln for ln in text.splitlines() if ln.startswith("> Revision"))
        numbers = [int(n) for n in re.findall(r"\*\*(\d+)\*\*", first)]
        self.assertIn(self.fs["sources"], numbers, first[:200])
        self.assertIn(self.fs["symbols"], numbers, first[:200])


class RepositoryToolingHygiene(unittest.TestCase):
    def test_every_python_tool_compiles(self):
        bad = []
        for p in sorted((rs.ROOT / "tools").rglob("*.py")):
            try:
                compile(p.read_text(encoding="utf-8"), str(p), "exec")
            except (SyntaxError, UnicodeDecodeError) as e:
                bad.append(f"{p.relative_to(rs.ROOT)}: {e}")
        self.assertEqual(bad, [])


class CompilerSelection(unittest.TestCase):
    """Regression (was HARNESS-RELATIVE-SLEELA-BIN): a relative $SLEELA_BIN was
    returned as given, and the CLI suites run the compiler from scratch
    directories, so nearly every CLI test failed with FileNotFoundError. A
    missing $SLEELA_BIN silently fell back to a different compiler."""

    def setUp(self):
        self.saved = os.environ.get("SLEELA_BIN")
        self.cwd = os.getcwd()

    def tearDown(self):
        os.chdir(self.cwd)
        if self.saved is None:
            os.environ.pop("SLEELA_BIN", None)
        else:
            os.environ["SLEELA_BIN"] = self.saved

    def test_relative_path_is_resolved_to_an_absolute_one(self):
        with tempfile.TemporaryDirectory() as tmp:
            exe = Path(tmp) / "bin" / "fake-sleela"
            exe.parent.mkdir()
            exe.write_text("#!/bin/sh\nexit 0\n")
            exe.chmod(0o755)
            os.chdir(tmp)
            os.environ["SLEELA_BIN"] = "bin/fake-sleela"
            got = rs.sleela_binary()
            self.assertTrue(got.is_absolute(), got)
            self.assertEqual(got, exe.resolve())

    def test_explicit_path_that_is_not_an_executable_is_an_error(self):
        with tempfile.TemporaryDirectory() as tmp:
            not_exec = Path(tmp) / "plain.txt"
            not_exec.write_text("x")
            not_exec.chmod(0o644)
            for value in (str(Path(tmp) / "missing"), str(not_exec), tmp):
                os.environ["SLEELA_BIN"] = value
                with self.assertRaises(rs.CompilerSelectionError, msg=value):
                    rs.sleela_binary()

    def test_runner_refuses_to_start_with_a_bad_sleela_bin(self):
        env = dict(os.environ, SLEELA_BIN="/nonexistent/sleela")
        r = subprocess.run([sys.executable, str(rs.HERE / "run_regression.py"), "--only", "integrity"],
                           capture_output=True, text=True, env=env, timeout=120)
        self.assertEqual(r.returncode, 2, r.stdout + r.stderr)
        self.assertIn("SLEELA_BIN", r.stderr)


if __name__ == "__main__":
    unittest.main()