rune/scripts/check_response_adapter_parity.py

#!/usr/bin/env python3
"""Live parity: response_adapter Rust core vs Python oracle (dichotomy
wave, close-PARTIAL gap pass). Layers:

  1. FROZEN — the gap truth corpus (src/rust/response_adapter/
     frozen_ra_gap_truth.json, generated by
     scripts/gen_response_adapter_truth.py against the LIVE Python oracle
     through the scripts/ra_freeze_seams rig: pinned
     RUNE_RESPONSE_ADAPTER_CORE=python floor, to_dict-carrying wrappers)
     replayed through the ra_replay ctypes FFI entry of the built cdylib,
     case by case, with ORDER-SENSITIVE comparison (json.dumps preserves
     the insertion order on both sides — a key-order divergence fails).
  2. FUZZ — seeded random responses (seed 2479): random content built
     from token fragments including <think>/</think>, Thinking Process:/
     Final:, unicode, whitespace runs; random usage dicts (ints, floats,
     nulls), model/provider strings. The oracle's parse fns (pin ON, pure
     Python) vs the Rust core through the FFI, exact ordered-dict compare
     of the to_dict forms, across explicit and auto providers.
  3. PINS — dual-mode facade battery: the same facade calls executed with
     RUNE_RESPONSE_ADAPTER_CORE=python (oracle floor) and with the Rust
     core live (delegation ON); results must be identical (the facade
     both-pins law).

Exit 0 only at 100% parity.
"""
import json
import os
import random
import subprocess
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
CRATE = REPO / "src" / "rust" / "response_adapter"
FROZEN = CRATE / "frozen_ra_gap_truth.json"

os.environ["RUNE_RESPONSE_ADAPTER_CORE"] = "python"  # oracle floor to start

sys.path.insert(0, str(REPO / "src"))

import rune.response_adapter as ra  # noqa: E402
import rune.ra_loader as core  # noqa: E402

if not core.available():
    sys.exit(f"FATAL: Rust core not built — cargo build --release in {CRATE}")

TRUTH = json.loads(FROZEN.read_text())

fails: list[str] = []
counts = {"frozen": 0, "fuzz": 0, "pins": 0}


def fail(layer, name, detail):
    fails.append(f"[{layer}] {name}: {detail}")


def ordered(obj) -> str:
    return json.dumps(obj, ensure_ascii=False, separators=(", ", ": "))


def frozen_layer() -> None:
    for i, case in enumerate(TRUTH):
        try:
            got = core.replay(case)
        except Exception as e:  # noqa: BLE001
            fail("FROZEN", f"case{i}:{case['fn']}", f"FFI error {e}")
            continue
        want = case["expected"]
        if ordered(got) != ordered(want):
            fail("FROZEN", f"case{i}:{case['fn']}",
                 f"\n  rust: {ordered(got)[:200]}\n  want: {ordered(want)[:200]}")
        else:
            counts["frozen"] += 1


FRAGMENTS = [
    "<think>", "</think>", "Thinking Process:", "Final:", "hello", "wörld",
    "答案", "🜏", "  ", "\n", "\t", "a b", "x", "", "Final", "<think >",
    "process:", "think", "…", "\r\n", " three   spaces ", "Final:  ",
]
MODELS = ["mimo-v2.5-pro", "qwen3:14b", "", "モデル", None]
PROVIDERS = [None, "mimo", "qwen", "local", "plain", "weird"]


def rand_response(rng) -> dict:
    content = "".join(rng.choice(FRAGMENTS)
                      for _ in range(rng.randint(0, 14)))
    resp: dict = {"content": content}
    m = rng.choice(MODELS)
    if m is not None:
        resp["model"] = m
    if rng.random() < 0.5:
        resp["reasoning_content"] = "".join(
            rng.choice(FRAGMENTS) for _ in range(rng.randint(0, 6)))
    if rng.random() < 0.5:
        usage: dict = {}
        for key in ("reasoning_tokens", "completion_tokens", "total_tokens"):
            r = rng.random()
            if r < 0.4:
                usage[key] = rng.randint(0, 5000)
            elif r < 0.5:
                usage[key] = float(rng.randint(0, 90)) / 3
            elif r < 0.55:
                usage[key] = None
        resp["usage"] = usage
    if rng.random() < 0.3:
        resp["provider"] = rng.choice(["openai", "groq", ""])
    return resp


def fuzz_layer(n_seed: int = 400) -> None:
    rng = random.Random(2479)
    parse_fns = [
        ("parse_mimo_response", ra.parse_mimo_response,
         core.parse_mimo_response),
        ("parse_qwen_response", ra.parse_qwen_response,
         core.parse_qwen_response),
        ("parse_local_response", ra.parse_local_response,
         core.parse_local_response),
        ("parse_plain_response", ra.parse_plain_response,
         core.parse_plain_response),
    ]
    for i in range(n_seed):
        resp = rand_response(rng)
        for name, py_fn, rs_fn in parse_fns:
            want = py_fn(dict(resp)).to_dict()
            try:
                got = rs_fn(dict(resp))
            except Exception as e:  # noqa: BLE001
                fail("FUZZ", f"{i}:{name}", f"FFI error {e}")
                continue
            if ordered(got) != ordered(want):
                fail("FUZZ", f"{i}:{name}",
                     f"resp={json.dumps(resp, ensure_ascii=False)}\n"
                     f"  rust: {ordered(got)[:220]}\n  want: {ordered(want)[:220]}")
            else:
                counts["fuzz"] += 1
        provider = rng.choice(PROVIDERS)
        want = ra.parse_response(dict(resp), provider).to_dict()
        try:
            got = core.parse_response(dict(resp), provider)
        except Exception as e:  # noqa: BLE001
            fail("FUZZ", f"{i}:parse_response", f"FFI error {e}")
            continue
        if ordered(got) != ordered(want):
            fail("FUZZ", f"{i}:parse_response",
                 f"resp={json.dumps(resp, ensure_ascii=False)} provider={provider}\n"
                 f"  rust: {ordered(got)[:220]}\n  want: {ordered(want)[:220]}")
        else:
            counts["fuzz"] += 1


PIN_SNIPPET = r"""
import json, os, sys
sys.path.insert(0, "src")
import rune.response_adapter as ra
samples = [
    {"reasoning_content": "think hard", "content": "mimo answer",
     "model": "mimo-v2.5-pro",
     "usage": {"reasoning_tokens": 4, "completion_tokens": 3, "total_tokens": 7}},
    {"content": "<think>one two three</think>\n\n  the answer  ",
     "model": "qwen3:14b", "usage": {"completion_tokens": 20}},
    {"content": "Thinking Process: reason it out\nline two Final:  final text",
     "model": "qwen2.5:3b"},
    {"content": "plain words", "provider": "openai", "model": "gpt-4"},
    {"content": "auto <think>x</think> shape"},
    {"content": "auto Thinking Process: y Final: z"},
]
out = [ra.parse_response(s).to_dict() for s in samples]
out += [ra.parse_mimo_response(samples[0]).to_dict(),
        ra.parse_qwen_response(samples[1]).to_dict(),
        ra.parse_local_response(samples[2]).to_dict(),
        ra.parse_plain_response(samples[3]).to_dict()]
print(json.dumps(out, ensure_ascii=False))
"""


def pins_layer() -> None:
    results = {}
    for pin in ("python", "rust"):
        env = dict(os.environ)
        env.pop("RUNE_RESPONSE_ADAPTER_CORE", None)
        if pin == "python":
            env["RUNE_RESPONSE_ADAPTER_CORE"] = "python"
        env["RA_RUST_LIB"] = str(CRATE / "target" / "release" /
                                 "libresponse_adapter.so")
        r = subprocess.run([sys.executable, "-c", PIN_SNIPPET],
                           cwd=str(REPO), env=env, capture_output=True,
                           text=True, timeout=120)
        if r.returncode != 0:
            fail("PINS", pin, f"snippet failed:\n{r.stderr[-400:]}")
            return
        results[pin] = r.stdout
    if results["python"] != results["rust"]:
        fail("PINS", "both-pins", "python-pinned vs live-core outputs differ")
    else:
        counts["pins"] = len(json.loads(results["python"]))


if __name__ == "__main__":
    frozen_layer()
    fuzz_layer()
    pins_layer()
    print(f"FROZEN: {counts['frozen']}/{len(TRUTH)} cases verbatim through ra_replay")
    print(f"FUZZ:   {counts['fuzz']} op comparisons (seed 2479)")
    print(f"PINS:   {counts['pins']} facade calls identical under both pins")
    if fails:
        print(f"\n{len(fails)} FAILURES (first 8):")
        for f in fails[:8]:
            print(" -", f)
        sys.exit(1)
    print("PARITY 100%")