rune/scripts/check_model_connect_parity.py

#!/usr/bin/env python3
"""check_model_connect_parity.py — live oracle-vs-Go parity sweep for the
model_connect gap closure (#2479, contract factory-gap-mconn).

The frozen envelope (truth.json) pins the ported surface byte-identically.
THIS script drives the LIVE Python oracle and the Go binary (stdin op
dispatch) across every op family — clean, moe, route, ctx_budget, now
(clock pinned on both sides), hook, and the cloud-model config sweep
(per-fixture hermetic HOME shared by oracle and binary) — comparing
sort_keys JSON. Exit 0 on 100% parity, 1 with the first divergences.

Usage: python3 scripts/check_model_connect_parity.py
  [--bin src/go/model-connect/model-connect]
"""
import argparse
import json
import os
import subprocess
import sys
import tempfile
from datetime import datetime, timezone
from pathlib import Path

REPO = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO / "src"))

import rune.model_connect as mc  # noqa: E402

FROZEN_NOW = "2026-10-14T08:00:00+00:00"

CLEAN = [
    "hf.co/DavidAU/Qwen3.6-27B-Fable-Fusion-711-Uncensored-Heretic-NM-DAU-NEO-MAX-MTP-GGUF:IQ2_M",
    "hf.co/org/Llama-3-8B-Instruct:latest",
    "hf.co/short/gemma-2-9b",
    "hf.co/user/nano-model",
    "hf.co/user/ab",
    "qwen2.5:3b",
    "mimo-v2.5-pro",
    "llava:7b",
    "huggingface.co/x/y/Llama-3-8B",
]

MOE = [
    "mixtral:8x7b", "qwen2.5:3b", "deepseek-moe-16b", "My-Mixture-Model",
    "expert-v1", "4x22b-tiny", "plain-conversational", "GPT-XL",
]

MODELS_ALL = [
    "qwen2.5:3b", "qwen2.5:7b", "qwen3:14b", "qwen2.5-coder:14b",
    "hf.co/DavidAU/Qwen3.6-27B-Fable-Fusion-711-Uncensored-Heretic-NM-DAU-NEO-MAX-MTP-GGUF:IQ2_M",
    "mixtral:8x7b-instruct-v0.1-q4_0", "qwen2.5vl:3b", "llava:7b",
    "qwen3-embedding:0.6b", "nomic-embed-text:latest", "mxbai-embed-large:latest",
]

ROUTE = [
    ("coding", MODELS_ALL), ("research", MODELS_ALL), ("conversation", MODELS_ALL),
    ("unrestricted", MODELS_ALL), ("unknown-type", MODELS_ALL),
    ("coding", ["mystery-model:x"]), ("coding", []),
]

CTX = ["deterministic", "qwen2.5:3b", "qwen3:14b", "qwen3.6:27b",
       "mimo-v2.5-pro", "totally-unknown-model", ""]

HOOKS = [
    ("pre_tool_use", {"tool": "bash", "dry": False}),
    ("", None),
    ("post_response", {"ok": True}),
    ("unicode-évent", {}),
]

FIXTURES = {
    "main": """{
  // MiMoCode config — JSONC: comments and trailing commas allowed
  "provider": {
    "ollama": { // skipped: local models handled by list_models
      "models": {
        "qwen3:14b": {},
      },
    },
    "openai": {
      "models": {
        "gpt-5.2": {"contextLength": 400000},
        "o4-mini": "not-even-a-dict",
      },
    },
    "kimi": {
      "models": {
        "kimi-for-coding": {},
      },
    },
  },
}
""",
    "only_ollama": '{\n  "provider": {\n    "ollama": {"models": {"qwen2.5:3b": {}}},\n  },\n}\n',
    "no_models_key": '{\n  "provider": {\n    "anthropic": {"apiKey": "sk-test"},\n  },\n}\n',
    "no_provider_key": '{\n  "theme": "dark",\n}\n',
    "broken_late": '{\n  "provider": {\n    "a": {"models": {"m1": {}}},\n    "b": {"models": "not-a-dict"},\n  },\n}\n',
    "null_models": '{\n  "provider": {\n    "z": {"models": null},\n  },\n}\n',
    "url_corruption": '{\n  "provider": {\n    "gateway": {\n      "baseUrl": "http://api.example.com/v1",\n      "models": {"gw-model": {}},\n    },\n  },\n}\n',
}


def run_go(binpath, req, env=None):
    e = dict(os.environ)
    if env:
        e.update(env)
    p = subprocess.run([str(binpath)], input=json.dumps(req), capture_output=True,
                       text=True, env=e)
    if p.returncode != 0:
        raise RuntimeError(f"go op {req.get('op')} exited {p.returncode}: {p.stdout} {p.stderr}")
    return json.loads(p.stdout)


def frozen_clock():
    base = datetime.fromisoformat(FROZEN_NOW)

    class FrozenDatetime(datetime):
        @classmethod
        def now(cls, tz=None):
            return base if tz is not None else base.replace(tzinfo=None)

    return FrozenDatetime


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--bin", default=str(REPO / "src/go/model-connect/model-connect"))
    args = ap.parse_args()
    binpath = Path(args.bin)
    if not binpath.exists():
        subprocess.run(["go", "build", "-o", str(binpath), str(binpath.parent)],
                       check=True, cwd=binpath.parent)

    fails = total = 0

    def check(name, oracle_val, go_val):
        nonlocal fails, total
        total += 1
        if json.dumps(oracle_val, sort_keys=True) != json.dumps(go_val, sort_keys=True):
            fails += 1
            print(f"DIVERGE {name}:\n  oracle: {json.dumps(oracle_val)[:300]}\n  go:     {json.dumps(go_val)[:300]}")

    for name in CLEAN:
        g = run_go(binpath, {"op": "clean", "name": name})
        check(f"clean:{name[:40]}", {"result": mc._clean_model_name(name)}, g)

    for name in MOE:
        g = run_go(binpath, {"op": "moe", "name": name})
        check(f"moe:{name}", {"result": mc.is_moe_model(name)}, g)

    for task, models in ROUTE:
        g = run_go(binpath, {"op": "route", "task": task, "models": models})
        check(f"route:{task}", mc.route_model(task, models), g)

    for mid in CTX:
        g = run_go(binpath, {"op": "ctx_budget", "model": mid})
        check(f"ctx_budget:{mid}", {"result": mc._ctx_budget(mid)}, g)

    saved_dt = mc.datetime
    mc.datetime = frozen_clock()
    try:
        g = run_go(binpath, {"op": "now"}, env={"MCONN_CLOCK": FROZEN_NOW})
        check("now", {"result": mc._now()}, g)
    finally:
        mc.datetime = saved_dt

    for event, data in HOOKS:
        g = run_go(binpath, {"op": "hook", "event": event, "data": data})
        check(f"hook:{event}", {"result": mc.handle_hook(event, data)}, g)

    saved_home = os.environ.get("HOME")
    try:
        for label, fixture in FIXTURES.items():
            with tempfile.TemporaryDirectory() as td:
                cfg = os.path.join(td, ".config", "mimocode")
                os.makedirs(cfg)
                with open(os.path.join(cfg, "mimocode.jsonc"), "w") as f:
                    f.write(fixture)
                os.environ["HOME"] = td
                oracle_val = mc._get_cloud_models()
                g = run_go(binpath, {"op": "cloud_models"}, env={"HOME": td})
                check(f"cloud:{label}", {"result": oracle_val}, g)
        with tempfile.TemporaryDirectory() as td:
            os.environ["HOME"] = td
            oracle_val = mc._get_cloud_models()
            g = run_go(binpath, {"op": "cloud_models"}, env={"HOME": td})
            check("cloud:missing", {"result": oracle_val}, g)
    finally:
        if saved_home is not None:
            os.environ["HOME"] = saved_home

    print(f"parity: {total - fails}/{total} match")
    return 1 if fails else 0


if __name__ == "__main__":
    sys.exit(main())