rune/scripts/check_rune_profiler_parity.py

#!/usr/bin/env python3
"""Live parity: rune_profiler Rust kernel vs Python oracle (dichotomy
wave). Layers:
  1. FROZEN — 9 frozen-oracle cases (controlled dyadic elapsed deltas)
     replayed through BOTH the Python floor and the Rust kernel; means
     are BIT-FOR-BIT.
  2. FUZZ   — 800 seeded random dyadic elapsed sequences (seed 2479)
     driven through the oracle with the same monkeypatched clock and
     through the kernel; exact compare.
  3. LIVE   — the real measured path: oracle floor vs facade(kernel) on
     the same workloads; timings are nondeterministic by nature so the
     comparison is STRUCTURE + INVARIANTS (key set, iterations, ordering
     min<=median<=max, p95/p99 within [median, max], mean within
     [min, max]).
"""
import itertools
import json
import os
import random
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / "src"))

import rune.rune_profiler as rp  # noqa: E402
from rune import rp_loader  # noqa: E402

UNIT_NS = 15625  # 1/64 ms — dyadic

REAL_CLOCK = rp.time.perf_counter_ns
REAL_OLLAMA = rp._execute_ollama

TRUTH = json.loads(
    (REPO / "src" / "rust" / "rune_profiler" / "src" / "frozen_rp_truth.json")
    .read_text())

fails = []
counts = {"frozen": 0, "fuzz": 0, "live": 0}


def fail(layer, name, detail):
    fails.append((layer, name, detail))
    print(f"FAIL[{layer}] {name}: {detail}")


def drive_oracle(elapsed_units, iterations, model_call):
    vals, c = [], 0
    for u in elapsed_units:
        vals.append(c)
        vals.append(c + u * UNIT_NS)
        c += u * UNIT_NS
    clock = iter(list(vals) + [c])
    rp.time.perf_counter_ns = lambda: next(clock)
    stub = itertools.count()
    rp._execute_ollama = lambda *a, **k: next(stub)
    if model_call:
        return rp.profile_model_call(iterations=iterations)
    return rp.profile_function(lambda: None, iterations=iterations)


def frozen():
    for group, cases in TRUTH.items():
        if group in ("module", "unit_ns"):
            continue
        for c in cases:
            counts["frozen"] += 1
            args = c["args"]
            model_call = group == "profile_model_call_cases"
            units = [round(t * 64) for t in args["times"]]
            want = c["expect"]
            got_o = drive_oracle(units, args["iterations"], model_call)
            got_k = rp_loader.profile_stats(args["times"],
                                            args["iterations"], model_call)
            for tag, got in (("oracle", got_o), ("kernel", got_k)):
                if json.dumps(got, sort_keys=True) != json.dumps(
                        want, sort_keys=True):
                    fail("frozen", f"{c['name']}/{tag}",
                         f"want={json.dumps(want, sort_keys=True)[:260]} "
                         f"got={json.dumps(got, sort_keys=True)[:260]}")


def fuzz():
    rnd = random.Random(2479)
    for i in range(800):
        counts["fuzz"] += 1
        n = rnd.choice([1, 2, 3, 5, 20, 100, 101])
        model_call = rnd.random() < 0.4
        units = [(rnd.randint(1, 8192))] * 0 or \
                [rnd.randint(1, 8192) for _ in range(n)]
        want = drive_oracle(units, n, model_call)
        got = rp_loader.profile_stats([u * UNIT_NS / 1_000_000
                                       for u in units], n, model_call)
        if json.dumps(want, sort_keys=True) != json.dumps(
                got, sort_keys=True):
            fail("fuzz", f"fuzz{i}", f"oracle={want} kernel={got}")


def invariants(stats, n, model_call):
    keys = {"min_ms", "max_ms", "mean_ms", "median_ms", "iterations"}
    if not model_call:
        keys |= {"p95_ms", "p99_ms"}
    if set(stats) != keys:
        return f"key set {sorted(stats)}"
    if stats["iterations"] != n:
        return f"iterations {stats['iterations']} != {n}"
    if not (stats["min_ms"] <= stats["median_ms"] <= stats["max_ms"]):
        return "min<=median<=max violated"
    if not (stats["min_ms"] <= stats["mean_ms"] <= stats["max_ms"]):
        return "mean out of range"
    if not model_call:
        if not (stats["median_ms"] <= stats["p95_ms"] <= stats["p99_ms"]
                <= stats["max_ms"]):
            return "percentile ordering violated"
    return None


def live():
    rp.time.perf_counter_ns = REAL_CLOCK
    rp._execute_ollama = REAL_OLLAMA
    for name, fn, n, mc in [
        ("salience", lambda: sum(range(50)), 50, False),
        ("trivial", lambda: None, 25, False),
        ("model_shape", lambda: "ok", 5, True),
    ]:
        os.environ["RUNE_RP_CORE"] = "python"
        o = rp.profile_function(fn, iterations=n)
        om = rp.profile_model_call(iterations=3)
        os.environ["RUNE_RP_CORE"] = "auto"
        k = rp.profile_function(fn, iterations=n)
        km = rp.profile_model_call(iterations=3)
        for tag, a, b, iters, model in [
            (name, o, k, n, False),
            ("model", om, km, 3, True),
        ]:
            counts["live"] += 1
            probs = [invariants(a, iters, model),
                     invariants(b, iters, model)]
            if json.dumps(list(a), sort_keys=True) != \
                    json.dumps(list(b), sort_keys=True):
                probs.append("key order differs")
            probs = [p for p in probs if p]
            if probs:
                fail("live", tag, "; ".join(probs))


def main():
    if not rp_loader.available():
        print("kernel not built — build src/rust/rune_profiler first")
        return 1
    os.environ["RUNE_RP_CORE"] = "python"
    frozen()
    fuzz()
    live()
    os.environ["RUNE_RP_CORE"] = "auto"
    total = counts["frozen"] + counts["fuzz"] + counts["live"]
    print(f"parity {total - len(fails)}/{total} "
          f"(frozen {counts['frozen']}, fuzz {counts['fuzz']}, "
          f"live {counts['live']})")
    return 1 if fails else 0


if __name__ == "__main__":
    sys.exit(main())