156 lines
6.7 KiB
Python
156 lines
6.7 KiB
Python
"""Live parity probe: model_routing GO vs LIVE python oracle.
|
|
|
|
Generates fresh seeded inputs (seed 2479-live, distinct from the freeze
|
|
seed), evaluates them against the live oracle AND via the Go replay
|
|
harness, and reports exact-match parity. Run AFTER `go build -o
|
|
/tmp/model_routing_probe .` (rebuilds the current kernel).
|
|
"""
|
|
|
|
import json
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import os
|
|
|
|
sys.path.insert(0, "src")
|
|
from rune import model_routing as mr
|
|
|
|
N_LOAD = 150
|
|
N_RANK = 200
|
|
N_UPDATE = 150
|
|
|
|
rng = random.Random(24791)
|
|
|
|
ops = []
|
|
|
|
# live load probes
|
|
for i in range(N_LOAD):
|
|
rows = []
|
|
for _ in range(rng.randint(0, 4)):
|
|
row = {"provider": rng.choice(["ollama", "openai", "cloud", "", 5, None]),
|
|
"model": rng.choice(["q:3b", "mimo-v2.5-pro", "", None]),
|
|
"per_family_success": {f: [rng.randint(-1, 5), rng.randint(0, 6)]
|
|
for f in rng.sample(["a", "b", "c", "d"], rng.randint(1, 3))},
|
|
"avg_latency_ms": rng.choice([0.0, 1.5, 99.5, 123.25]),
|
|
"estimated_cost_usd": rng.choice([0.0, 2.5, 15.0, 7.75])}
|
|
if rng.random() < 0.25:
|
|
row["per_family_success"] = rng.choice([
|
|
{"g": [True, 2]}, {"g": [1.5, 2]}, {"g": ["x", 2]}, "junk",
|
|
{"g": [1, 2, 3]}, {3: [1, 2]}])
|
|
if rng.random() < 0.15:
|
|
row["avg_latency_ms"] = rng.choice(["abc", None, True])
|
|
rows.append(row)
|
|
content = json.dumps(rows)
|
|
if rng.random() < 0.08 and len(content) > 3:
|
|
content = content[: rng.randint(1, len(content) - 2)]
|
|
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
|
|
f.write(content)
|
|
path = f.name
|
|
reg = mr.load_profiles(path)
|
|
os.unlink(path)
|
|
ops.append({"op": "load", "file": content,
|
|
"expect": [{"provider": p.provider, "model": p.model,
|
|
"families": [[f, list(v)] for f, v in p.per_family_success.items()],
|
|
"avg_latency_ms": p.avg_latency_ms,
|
|
"estimated_cost_usd": p.estimated_cost_usd}
|
|
for p in reg.values()]})
|
|
|
|
# live rank probes
|
|
for i in range(N_RANK):
|
|
cands = []
|
|
for _ in range(rng.randint(1, 5)):
|
|
cands.append([rng.choice(["q:3b", "q3:14b", "mimo-v2.5-pro", "cx", "zz"]),
|
|
rng.choice(["ollama", "openai", "cloud"]),
|
|
rng.choice(["T1", "T2", "T3", "T4", "T9", "T10", "TX"]),
|
|
rng.random() < 0.6,
|
|
rng.choice([1024, 32768, 128000, 500]),
|
|
rng.choice([0.0, 0.5, 1, 2.5, 15.0, 99.5])])
|
|
profiles = None
|
|
if rng.random() < 0.7:
|
|
profiles = {}
|
|
for _ in range(rng.randint(1, 4)):
|
|
fams = {f: [rng.choice([0, 0, 1, 2, -1]), rng.randint(0, 5)]
|
|
for f in rng.sample(["a", "b", "c"], rng.randint(1, 3))}
|
|
profiles[(rng.choice(["ollama", "openai", "cloud"]),
|
|
rng.choice(["q:3b", "q3:14b", "mimo-v2.5-pro", "cx", "zz"]))] = \
|
|
mr.MeasuredModelProfile(
|
|
provider="x", model="y", per_family_success=dict(fams),
|
|
avg_latency_ms=rng.choice([0.0, 4.5, 99.5]),
|
|
estimated_cost_usd=rng.choice([0.0, 1.5, 15.0]))
|
|
router = mr.ModelRouter(candidates=tuple(tuple(r) for r in cands), profiles=profiles)
|
|
tf = rng.choice(["a", "b", "c", "nomatch"])
|
|
nt = rng.random() < 0.4
|
|
mc = rng.choice([0, 0, 1000, 32768, 64000, 128000, 200000])
|
|
decisions = router.rank(tf, need_tools=nt, min_context=mc)
|
|
prof_json = None
|
|
if profiles is not None:
|
|
prof_json = []
|
|
for (prov, mod), p in profiles.items():
|
|
prof_json.append({"provider": prov, "model": mod,
|
|
"families": [[f, list(v)] for f, v in p.per_family_success.items()],
|
|
"avg_latency_ms": p.avg_latency_ms,
|
|
"estimated_cost_usd": p.estimated_cost_usd})
|
|
ops.append({"op": "rank",
|
|
"candidates": cands, "profiles": prof_json,
|
|
"task_family": tf, "need_tools": nt, "min_context": mc,
|
|
"expect": [{"model": d.model, "provider": d.provider, "tier": d.tier,
|
|
"reason": d.reason, "estimated_cost": d.estimated_cost}
|
|
for d in decisions]})
|
|
|
|
# live update probes
|
|
for i in range(N_UPDATE):
|
|
start = None
|
|
if rng.random() < 0.7:
|
|
rows = []
|
|
for _ in range(rng.randint(0, 3)):
|
|
rows.append({"provider": rng.choice(["ollama", "openai"]),
|
|
"model": rng.choice(["q:3b", "m"]),
|
|
"per_family_success": {f: [rng.randint(0, 3), rng.randint(1, 5)]
|
|
for f in rng.sample(["a", "b"], rng.randint(1, 2))},
|
|
"avg_latency_ms": rng.choice([0.0, 3.5]),
|
|
"estimated_cost_usd": rng.choice([0.0, 1.25])})
|
|
start = json.dumps(rows)
|
|
traces = []
|
|
for _ in range(rng.randint(0, 6)):
|
|
t = {"provider": rng.choice(["ollama", "openai", "", None, 7]),
|
|
"model": rng.choice(["q:3b", "m", "", None]),
|
|
"success": rng.choice([True, False, 1, "true", None]),
|
|
"task_family": rng.choice(["a", "b", "", None, 0])}
|
|
if rng.random() < 0.15:
|
|
t.pop("task_family")
|
|
if rng.random() < 0.1:
|
|
t.pop("success")
|
|
traces.append(t)
|
|
argfam = rng.choice(["a", "b", ""])
|
|
d = tempfile.mkdtemp()
|
|
path = os.path.join(d, "profiles.json")
|
|
if start is not None:
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(start)
|
|
reg = mr.update_profiles_from_traces(path, traces, task_family=argfam)
|
|
out = os.path.join(d, "out.json")
|
|
mr.save_profiles(out, reg)
|
|
with open(out, "r", encoding="utf-8") as f:
|
|
raw = f.read()
|
|
prof_list = []
|
|
for p in reg.values():
|
|
prof_list.append({"provider": p.provider, "model": p.model,
|
|
"families": [[f, list(v)] for f, v in p.per_family_success.items()],
|
|
"avg_latency_ms": p.avg_latency_ms,
|
|
"estimated_cost_usd": p.estimated_cost_usd})
|
|
ops.append({"op": "update", "file": start, "traces": traces, "task_family": argfam,
|
|
"expect_bytes": raw, "expect_profiles": prof_list})
|
|
|
|
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
|
|
json.dump(ops, f)
|
|
probe_path = f.name
|
|
|
|
result = subprocess.run(
|
|
["/tmp/model_routing_probe", "live", probe_path],
|
|
capture_output=True, text=True)
|
|
os.unlink(probe_path)
|
|
print(result.stdout[-4000:])
|
|
print(result.stderr[-2000:])
|
|
sys.exit(result.returncode)
|