Home /Modules/comparison/run_sleela_vm_arithmetic_speed.py

run_sleela_vm_arithmetic_speed.py

comparison/tests/run_sleela_vm_arithmetic_speed.py · 119 line(s) · py

#!/usr/bin/env python3
"""Steady-state int/double arithmetic timings inside Basic Native and Premium SLVM 1-11."""
from __future__ import annotations
import json, os, pathlib, platform, re, statistics, subprocess, sys

ROOT = pathlib.Path(__file__).resolve().parents[2]
CLI = pathlib.Path(os.environ.get("SLEELA_BIN", ROOT / "impl" / "build" / ("sleela.exe" if os.name == "nt" else "sleela")))
SOURCE = ROOT / "comparison" / "tests" / "sleela" / "arithmetic_speed.sleela"
OPS = ("INT_ADD", "INT_SUB", "INT_MUL", "INT_DIV",
       "DOUBLE_ADD", "DOUBLE_SUB", "DOUBLE_MUL", "DOUBLE_DIV")
CHECK_SCALE = 1_000_000  # Six decimal places, emitted as an exact integer.
EXPECTED = {"INT_ADD": 8888888, "INT_SUB": 7530865, "INT_MUL": 83810205,
            "INT_DIV": 10802469, "DOUBLE_ADD": 1_999_999_100,
            "DOUBLE_SUB": 7_530_865_100, "DOUBLE_MUL": 9_740_678_400,
            "DOUBLE_DIV": 10_802_469_000}
TIMING_RE = re.compile(r"(INT|DOUBLE)_(ADD|SUB|MUL|DIV)_NS=(\d+)")
CHECK_RE = re.compile(
    r"(INT|DOUBLE)_(ADD|SUB|MUL|DIV)_CHECK="
    r"([+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?)"
)
RESULT = {
    "suite": "SCS-PERF-SLEELA-VM-001", "status": "RUNNING",
    "host": {"os": platform.platform(), "machine": platform.machine(),
             "processor": platform.processor(), "logical_cores": os.cpu_count()},
    "measurement": {"iterations_per_case": 250000, "warmup_iterations": 50000,
                    "samples": 5, "clock": "timeMonotonicNanos",
                    "startup_and_admission_in_timed_region": False},
    "targets": {}
}


def invoke(label, command):
    p = subprocess.run(command, cwd=ROOT, text=True, stdout=subprocess.PIPE,
                       stderr=subprocess.STDOUT, timeout=240)
    if p.returncode:
        raise RuntimeError(f"{label} exited {p.returncode}:\n{p.stdout}")
    timings, checks = {}, {}
    for line in p.stdout.replace("\r\n", "\n").splitlines():
        line = line.strip()
        tm = TIMING_RE.fullmatch(line)
        if tm:
            key = tm.group(1) + "_" + tm.group(2)
            if key in timings:
                raise AssertionError(f"{label}: duplicate timing record {key}")
            timings[key] = int(tm.group(3))
            continue
        cm = CHECK_RE.fullmatch(line)
        if cm:
            key = cm.group(1) + "_" + cm.group(2)
            if key in checks:
                raise AssertionError(f"{label}: duplicate correctness record {key}")
            checks[key] = float(cm.group(3))
    missing_timings = sorted(set(OPS) - set(timings))
    missing_checks = sorted(set(OPS) - set(checks))
    if missing_timings:
        raise AssertionError(f"{label}: missing timing records {missing_timings}; output:\n{p.stdout}")
    if missing_checks:
        raise AssertionError(f"{label}: missing correctness records {missing_checks}; output:\n{p.stdout}")
    for op in OPS:
        ns = timings[op]
        if ns <= 0:
            raise AssertionError(f"{label}: non-positive timing for {op}: {ns}")
        expected = EXPECTED[op]
        # Floating values are scaled and rounded inside the SLeeLa workload,
        # then printed as integers. This avoids the runtime double-to-text %g
        # formatting limit while validating the result to 1e-6 precision.
        actual = int(checks[op])
        if actual != expected:
            raise AssertionError(
                f"{label}: {op} scaled checksum {actual} != {expected} "
                f"(scale {CHECK_SCALE}, 1e-6 resolution)"
            )
    return timings, checks, p.stdout


def main():
    if not CLI.is_file() or not SOURCE.is_file():
        raise FileNotFoundError(f"Missing SLeeLa CLI or arithmetic workload: {CLI}, {SOURCE}")
    targets = [("Basic Native", [str(CLI), "run", str(SOURCE)])]
    targets += [(f"Premium SLVM {n}", [str(CLI), "vm", "run", "premium", str(n), str(SOURCE)])
                for n in range(1, 12)]
    for label, command in targets:
        samples = {op: [] for op in OPS}
        checks_by_op = {op: [] for op in OPS}
        for _ in range(5):
            measured, checks, _ = invoke(label, command)
            for op in OPS:
                samples[op].append(measured[op])
                checks_by_op[op].append(checks[op])
        RESULT["targets"][label] = {
            op: {"samples_ns": values, "median_ns": statistics.median(values),
                 "median_ns_per_operation": statistics.median(values) / 250000,
                 "checksum": checks_by_op[op][0],
                 "checksum_samples": checks_by_op[op],
                 "status": "PASS"}
            for op, values in samples.items()
        }
        print(f"PASS {label}: 8 timing and checksum cases x 5 samples", flush=True)
    RESULT["status"] = "PASS"
    RESULT["counts"] = {"targets": 12, "premium_vms": 11, "types": 2,
                        "operators": 4, "measurements": 12 * 2 * 4 * 5,
                        "validated_checksums": 12 * 2 * 4 * 5}
    out = ROOT / "comparison-sleela-vm-arithmetic-results.json"
    out.write_text(json.dumps(RESULT, indent=2) + "\n", encoding="utf-8")
    print("PASS SLeeLa VM arithmetic:", out.name)
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except Exception as exc:
        RESULT["status"] = "FAIL"
        RESULT["error"] = str(exc)
        (ROOT / "comparison-sleela-vm-arithmetic-results.json").write_text(
            json.dumps(RESULT, indent=2) + "\n", encoding="utf-8")
        print(f"FAIL SLeeLa VM arithmetic: {exc}", file=sys.stderr)
        raise