606 lines
29 KiB
Python
606 lines
29 KiB
Python
"""Freeze truth for the model_routing GO port (issue #2479 dichotomy wave).
|
|
|
|
Exercises EVERY public function of src/rune/model_routing.py against the
|
|
LIVE python oracle with real inputs (all branches) and writes
|
|
src/go/model_routing/truth.json. Inputs AND expected outputs are both
|
|
committed — the Go battery replays this file verbatim; a divergence fails
|
|
the port, not the oracle.
|
|
|
|
Note (poison-truth law): model_routing.py has no RUNE_*_CORE kernel
|
|
switch — it is a pure python module (no facade delegation), so there is
|
|
no core pin to set. Verified 2026-10-11 before freezing.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import random
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
|
|
sys.path.insert(0, "src")
|
|
from rune import model_routing as mr # the oracle
|
|
|
|
cases = []
|
|
|
|
|
|
def case(fn, name, inp, expected):
|
|
cases.append({"fn": fn, "name": name, "input": inp, "expected": expected})
|
|
|
|
|
|
# ── capability floor + policy name ──────────────────────────────────────────
|
|
case("capability_floor", "floor_constant", {},
|
|
{"rows": [list(row) for row in mr.CAPABILITY_FLOOR]})
|
|
case("policy_name", "policy_name", {}, {"policy_name": mr.ModelRouter.policy_name})
|
|
|
|
# ── _tier_number ────────────────────────────────────────────────────────────
|
|
for i, tier in enumerate(["T1", "T2", "T10", "T12", "T0", "X1", "", "T", "Tabc", "t3", None, "T-2"]):
|
|
case("tier_number", f"tier_{i}", {"tier": tier}, {"want": mr._tier_number(tier)})
|
|
|
|
# ── default_profiles_path ───────────────────────────────────────────────────
|
|
for i, home in enumerate(["/rune-home-a", "/tmp/rh b", "rel-home", ""]):
|
|
old = os.environ.get("RUNE_HOME")
|
|
os.environ["RUNE_HOME"] = home
|
|
try:
|
|
want = str(mr.default_profiles_path())
|
|
finally:
|
|
if old is None:
|
|
os.environ.pop("RUNE_HOME", None)
|
|
else:
|
|
os.environ["RUNE_HOME"] = old
|
|
case("default_profiles_path", f"home_{i}", {"home": home}, {"path": want})
|
|
|
|
|
|
# ── profile JSON representation helpers ─────────────────────────────────────
|
|
def prof_json(p):
|
|
return {
|
|
"provider": p.provider,
|
|
"model": p.model,
|
|
"families": [[f, v[0], v[1]] for f, v in p.per_family_success.items()],
|
|
"avg_latency_ms": p.avg_latency_ms,
|
|
"estimated_cost_usd": p.estimated_cost_usd,
|
|
}
|
|
|
|
|
|
def profs_json(registry):
|
|
out = []
|
|
for p in registry.values():
|
|
fams = [[f, list(v)] for f, v in p.per_family_success.items()]
|
|
out.append({
|
|
"provider": p.provider,
|
|
"model": p.model,
|
|
"families": fams,
|
|
"avg_latency_ms": p.avg_latency_ms,
|
|
"estimated_cost_usd": p.estimated_cost_usd,
|
|
})
|
|
return out
|
|
|
|
|
|
# ── load_profiles ───────────────────────────────────────────────────────────
|
|
def load_case(name, content):
|
|
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
|
|
if content is not None:
|
|
f.write(content)
|
|
path = f.name
|
|
reg = mr.load_profiles(path)
|
|
os.unlink(path)
|
|
case("load_profiles", name, {"file": content}, {"profiles": profs_json(reg)})
|
|
|
|
|
|
load_case("missing_file", None)
|
|
load_case("empty_file", "")
|
|
load_case("garbage_json", "{not json")
|
|
load_case("not_a_list", '{"provider": "ollama"}')
|
|
load_case("not_a_list_number", "42")
|
|
load_case("empty_list", "[]")
|
|
load_case("row_not_dict", '[["ollama", "m"]]')
|
|
load_case("minimal_row", '[{"provider": "ollama", "model": "q:3b", "per_family_success": {"fix": [3, 4]}}]')
|
|
load_case("defaults_row", '[{"provider": "ollama", "model": "q:3b", "per_family_success": {"fix": [1, 2]}}]')
|
|
load_case("missing_stats", '[{"provider": "ollama", "model": "q"}]')
|
|
load_case("stats_not_dict", '[{"provider": "o", "model": "m", "per_family_success": [1, 2]}]')
|
|
load_case("family_stats_garbage_mixed",
|
|
'[{"provider": "o", "model": "m", "per_family_success": '
|
|
'{"ok": [2, 3], "bool": [true, 2], "float": [1.0, 2], "short": [1], "long": [1, 2, 3], '
|
|
'"strnum": ["1", 2], "nullpair": null}}]')
|
|
load_case("all_garbage_row_dropped",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"bool": [true, 2]}}]')
|
|
load_case("bool_stats_rejected",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"f": [true, true]}}]')
|
|
load_case("float_stats_rejected",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"f": [2.0, 4.0]}}]')
|
|
load_case("negative_stats_kept",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"f": [-1, 2]}}]')
|
|
load_case("zero_n_kept", '[{"provider": "o", "model": "m", "per_family_success": {"f": [3, 0]}}]')
|
|
load_case("provider_coerced_str", '[{"provider": 123, "model": null, "per_family_success": {"f": [1, 1]}}]')
|
|
load_case("family_coerced_str", '[{"provider": "o", "model": "m", "per_family_success": {7: [1, 1]}}]')
|
|
load_case("latency_string_number",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"f": [1, 1]}, "avg_latency_ms": "12.5"}]')
|
|
load_case("latency_bool",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"f": [1, 1]}, "estimated_cost_usd": true}]')
|
|
load_case("latency_garbage_drops_row",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"f": [1, 1]}, "avg_latency_ms": "abc"}]')
|
|
load_case("latency_null_drops_row",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"f": [1, 1]}, "avg_latency_ms": null}]')
|
|
load_case("duplicate_key_last_wins",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"f": [1, 1]}}, '
|
|
'{"provider": "o", "model": "m", "per_family_success": {"g": [2, 2]}}]')
|
|
load_case("two_providers_order",
|
|
'[{"provider": "zeta", "model": "m", "per_family_success": {"f": [1, 1]}}, '
|
|
'{"provider": "alpha", "model": "m", "per_family_success": {"f": [2, 2]}}]')
|
|
load_case("bom_file", "\ufeff[]")
|
|
load_case("unicode_family",
|
|
'[{"provider": "o", "model": "m", "per_family_success": {"caf\\u00e9": [1, 1]}}]')
|
|
|
|
# ── save_profiles (byte-exact JSON) ─────────────────────────────────────────
|
|
def save_case(name, profiles):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
path = os.path.join(d, "state", "nested", "profiles.json")
|
|
out = mr.save_profiles(path, {(p.provider, p.model): p for p in profiles})
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
raw = f.read()
|
|
reg2 = mr.load_profiles(path)
|
|
case("save_profiles", name,
|
|
{"profiles": profs_json({(p.provider, p.model): p for p in profiles})},
|
|
{"path": str(out), "bytes": raw, "profiles": profs_json(reg2)})
|
|
|
|
|
|
def mk(provider, model, fams, avg=0.0, cost=0.0):
|
|
return mr.MeasuredModelProfile(
|
|
provider=provider, model=model, per_family_success=dict(fams),
|
|
avg_latency_ms=avg, estimated_cost_usd=cost)
|
|
|
|
|
|
save_case("single_profile", [mk("ollama", "q:3b", {"fix": (3, 4)})])
|
|
save_case("empty_registry", [])
|
|
save_case("defaults_zero_floats", [mk("o", "m", {"f": (0, 0)})])
|
|
save_case("float_formatting", [mk("o", "m", {"f": (1, 2)}, avg=12.5, cost=15.0)])
|
|
save_case("float_small", [mk("o", "m", {"f": (1, 2)}, avg=0.5, cost=0.25)])
|
|
save_case("family_order_not_alpha", [mk("o", "m", {"zed": (1, 2), "alpha": (3, 4)})])
|
|
save_case("registry_order_preserved", [mk("zeta", "m", {"f": (1, 1)}), mk("alpha", "m", {"f": (2, 2)})])
|
|
save_case("negative_stats", [mk("o", "m", {"f": (-1, 2)}, avg=999.5)])
|
|
save_case("big_latency", [mk("o", "m", {"f": (1, 1)}, avg=123456789012345.0)])
|
|
|
|
# ── update_profiles_from_traces ─────────────────────────────────────────────
|
|
def update_case(name, start_content, traces, task_family=""):
|
|
d = tempfile.mkdtemp()
|
|
path = os.path.join(d, "profiles.json")
|
|
if start_content is not None:
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(start_content)
|
|
reg = mr.update_profiles_from_traces(path, traces, task_family=task_family)
|
|
out = os.path.join(d, "out.json")
|
|
mr.save_profiles(out, reg)
|
|
with open(out, "r", encoding="utf-8") as f:
|
|
raw = f.read()
|
|
case("update_traces", name,
|
|
{"file": start_content, "traces": traces, "task_family": task_family},
|
|
{"bytes": raw, "profiles": profs_json(reg)})
|
|
|
|
|
|
START = json.dumps([{"provider": "ollama", "model": "q:3b", "per_family_success": {"fix": [2, 3]},
|
|
"avg_latency_ms": 4.5, "estimated_cost_usd": 1.5}])
|
|
update_case("empty_traces", START, [])
|
|
update_case("missing_file_fresh", None, [{"provider": "o", "model": "m", "success": True, "task_family": "fix"}])
|
|
update_case("accumulate_existing", START,
|
|
[{"provider": "ollama", "model": "q:3b", "success": True}],
|
|
task_family="fix")
|
|
update_case("accumulate_fail", START,
|
|
[{"provider": "ollama", "model": "q:3b", "success": False}],
|
|
task_family="fix")
|
|
update_case("row_family_overrides_arg", START,
|
|
[{"provider": "ollama", "model": "q:3b", "success": True, "task_family": "rowfam"}],
|
|
task_family="argfam")
|
|
update_case("row_family_empty_falls_back", START,
|
|
[{"provider": "ollama", "model": "q:3b", "success": True, "task_family": ""}],
|
|
task_family="argfam")
|
|
update_case("row_family_null_falls_back", START,
|
|
[{"provider": "ollama", "model": "q:3b", "success": True, "task_family": None}],
|
|
task_family="argfam")
|
|
update_case("no_family_skipped", START,
|
|
[{"provider": "ollama", "model": "q:3b", "success": True}])
|
|
update_case("empty_provider_skipped", START,
|
|
[{"provider": "", "model": "m", "success": True, "task_family": "f"}])
|
|
update_case("empty_model_skipped", START,
|
|
[{"provider": "o", "model": "", "success": True, "task_family": "f"}])
|
|
update_case("missing_provider_skipped", START,
|
|
[{"model": "m", "success": True, "task_family": "f"}])
|
|
update_case("success_one_not_true", START,
|
|
[{"provider": "o", "model": "m", "success": 1, "task_family": "f"}])
|
|
update_case("success_string_not_true", START,
|
|
[{"provider": "o", "model": "m", "success": "true", "task_family": "f"}])
|
|
update_case("success_missing", START,
|
|
[{"provider": "o", "model": "m", "task_family": "f"}])
|
|
update_case("provider_coerced_from_none", START,
|
|
[{"provider": None, "model": "m", "success": True, "task_family": "f"}])
|
|
update_case("provider_coerced_from_int", START,
|
|
[{"provider": 7, "model": "m", "success": True, "task_family": "f"}])
|
|
update_case("family_zero_falsy", START,
|
|
[{"provider": "o", "model": "m", "success": True, "task_family": 0}],
|
|
task_family="argf")
|
|
update_case("family_string_zero", START,
|
|
[{"provider": "o", "model": "m", "success": True, "task_family": "0"}])
|
|
update_case("new_profile_defaults", None,
|
|
[{"provider": "o", "model": "m", "success": False, "task_family": "f"}])
|
|
update_case("multiple_models", START,
|
|
[{"provider": "ollama", "model": "q:3b", "success": True},
|
|
{"provider": "ollama", "model": "q3:14b", "success": True},
|
|
{"provider": "openai", "model": "mimo-v2.5-pro", "success": False}],
|
|
task_family="fix")
|
|
update_case("garbage_start_file", "{broken", [{"provider": "o", "model": "m", "success": True, "task_family": "f"}])
|
|
|
|
# ── adapters: chat normalization ────────────────────────────────────────────
|
|
CALL = {"model": "q:3b", "messages": [{"role": "user", "content": "hi"}],
|
|
"tools": [{"name": "t"}], "options": {"temperature": 0}, "think": True}
|
|
|
|
|
|
class FakeTransport:
|
|
def __init__(self, result, base_url=None, available=None):
|
|
self._result = result
|
|
self._available = available
|
|
if base_url is not None:
|
|
self.base_url = base_url
|
|
self.seen = None
|
|
|
|
def chat(self, *, model, messages, tools, options, think):
|
|
self.seen = {"model": model, "messages": messages, "tools": tools,
|
|
"options": options, "think": think}
|
|
return self._result
|
|
|
|
|
|
def chat_case(name, result, record_seen=False):
|
|
t = FakeTransport(result, base_url="http://127.0.0.1:1")
|
|
a = mr.OllamaAdapter(transport=t)
|
|
got = a.chat(**CALL)
|
|
expected = {"result": got}
|
|
if record_seen:
|
|
expected["transport_seen"] = t.seen
|
|
case("adapter_chat", name, {"transport_result": result, "call": CALL}, expected)
|
|
|
|
|
|
chat_case("plain_message_wrapped", {"role": "assistant", "content": "hello"})
|
|
chat_case("empty_message_wrapped", {})
|
|
chat_case("envelope_passthrough", {"message": {"role": "assistant", "content": "x"}, "done": True})
|
|
chat_case("envelope_empty_message_key", {"message": None})
|
|
chat_case("passthrough_with_extra_keys",
|
|
{"message": {"role": "assistant", "content": "y"}, "model": "q:3b", "done": False})
|
|
chat_case("kwargs_passthrough", {"role": "assistant", "content": "z"}, record_seen=True)
|
|
chat_case("think_string", {"role": "assistant"}, record_seen=True) # CALL reused; think variant below
|
|
# think variants (bool | str | None)
|
|
for i, think in enumerate([False, "medium", None]):
|
|
t = FakeTransport({"role": "assistant"})
|
|
a = mr.OllamaAdapter(transport=t)
|
|
got = a.chat(model="m", messages=[], tools=None, options=None, think=think)
|
|
case("adapter_chat", f"think_variant_{i}",
|
|
{"transport_result": {"role": "assistant"},
|
|
"call": {"model": "m", "messages": [], "tools": None, "options": None, "think": think}},
|
|
{"result": got, "transport_seen": t.seen})
|
|
|
|
# ── cloud adapter ───────────────────────────────────────────────────────────
|
|
cloud = mr.CloudStubAdapter()
|
|
chat_err = None
|
|
try:
|
|
cloud.chat(**CALL)
|
|
except mr.ProviderUnavailable as e:
|
|
chat_err = str(e)
|
|
case("cloud_adapter", "declared_not_live",
|
|
{},
|
|
{"name": cloud.name, "available": cloud.available(), "chat_error": chat_err,
|
|
"error_type": "ProviderUnavailable"})
|
|
|
|
# ── ollama adapter name ─────────────────────────────────────────────────────
|
|
case("ollama_name", "name", {}, {"name": mr.OllamaAdapter(transport=FakeTransport(None)).name})
|
|
|
|
# ── availability: direct probe (transport.available, no base_url) ──────────
|
|
for i, probe_vals in enumerate([[True], [False], [True, False, True], [0], [""], [None], ["yes"], [1]]):
|
|
t = FakeTransport(None) # unused for this branch
|
|
t2 = type("DirectProbe", (), {"available": staticmethod(lambda vals=iter(probe_vals): next(vals))})()
|
|
a = mr.OllamaAdapter(transport=t2)
|
|
wants = [a.available() for _ in probe_vals]
|
|
case("adapter_available", f"direct_probe_{i}",
|
|
{"kind": "direct", "probe_results": probe_vals}, {"wants": wants})
|
|
|
|
# ── availability: HTTP path with cache/TTL (base_url transports) ───────────
|
|
class _StatusHandler(BaseHTTPRequestHandler):
|
|
status = 200
|
|
|
|
def do_GET(self):
|
|
self.send_response(self.status)
|
|
self.end_headers()
|
|
self.wfile.write(b"{}")
|
|
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
|
|
def run_http_scenario(name, status, refused, also_available, steps):
|
|
if refused:
|
|
base = "http://127.0.0.1:1"
|
|
else:
|
|
srv = HTTPServer(("127.0.0.1", 0), _StatusHandler)
|
|
srv.RequestHandlerClass.status = status
|
|
th = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
th.start()
|
|
base = f"http://127.0.0.1:{srv.server_address[1]}"
|
|
attrs = {"base_url": base, "chat": lambda **kw: {}}
|
|
if also_available:
|
|
attrs["available"] = lambda: True # must NOT be consulted (has base_url)
|
|
t = type("HttpProbe", (), attrs)()
|
|
real_mono = time.monotonic
|
|
clock = {"t": 0.0}
|
|
time.monotonic = lambda: clock["t"]
|
|
try:
|
|
a = mr.OllamaAdapter(transport=t)
|
|
wants, counts = [], []
|
|
for step in steps:
|
|
clock["t"] = step["t"]
|
|
wants.append(a.available())
|
|
# a probe happened THIS call iff the cache was (re)written at the
|
|
# current fake-clock reading (cache timestamp == now).
|
|
counts.append(1 if a._available_cache[1] == clock["t"] else 0)
|
|
finally:
|
|
time.monotonic = real_mono
|
|
if not refused:
|
|
srv.shutdown()
|
|
srv.server_close()
|
|
case("adapter_available", name,
|
|
{"kind": "http", "status": status, "refused": refused,
|
|
"also_available": also_available,
|
|
"steps": [{"t": s["t"], "want": w, "want_probes": c}
|
|
for s, w, c in zip(steps, wants, counts)]},
|
|
{"wants": wants, "probe_counts": counts})
|
|
|
|
|
|
run_http_scenario("http_200_then_cache_then_expiry", 200, False, False,
|
|
[{"t": 0.0}, {"t": 10.0}, {"t": 29.9}, {"t": 30.0}, {"t": 40.0}])
|
|
run_http_scenario("http_500_cached_negative", 500, False, False,
|
|
[{"t": 0.0}, {"t": 5.0}, {"t": 30.5}, {"t": 31.0}])
|
|
run_http_scenario("http_refused", None, True, False,
|
|
[{"t": 0.0}, {"t": 100.0}])
|
|
run_http_scenario("http_available_ignored_when_base_url", 200, False, True,
|
|
[{"t": 0.0}, {"t": 31.0}])
|
|
|
|
# ── rank ────────────────────────────────────────────────────────────────────
|
|
def rank_case(name, candidates, profiles, task_family, need_tools, min_context,
|
|
profiles_path_content=None):
|
|
kw = {}
|
|
if candidates is not None:
|
|
kw["candidates"] = tuple(tuple(r) for r in candidates)
|
|
if profiles is not None:
|
|
kw["profiles"] = {(p["provider"], p["model"]):
|
|
mr.MeasuredModelProfile(
|
|
provider=p["provider"], model=p["model"],
|
|
per_family_success={f: [v[0], v[1]] for f, v in p["families"]},
|
|
avg_latency_ms=p["avg_latency_ms"],
|
|
estimated_cost_usd=p["estimated_cost_usd"])
|
|
for p in profiles}
|
|
d = None
|
|
if profiles_path_content is not None:
|
|
d = tempfile.mkdtemp()
|
|
pp = os.path.join(d, "p.json")
|
|
with open(pp, "w", encoding="utf-8") as f:
|
|
f.write(profiles_path_content)
|
|
kw["profiles_path"] = pp
|
|
router = mr.ModelRouter(**kw)
|
|
decisions = router.rank(task_family, need_tools=need_tools, min_context=min_context)
|
|
want = [{"model": x.model, "provider": x.provider, "tier": x.tier,
|
|
"reason": x.reason, "estimated_cost": x.estimated_cost}
|
|
for x in decisions]
|
|
inp = {"candidates": candidates, "profiles": profiles,
|
|
"file": profiles_path_content, "task_family": task_family,
|
|
"need_tools": need_tools, "min_context": min_context}
|
|
case("rank", name, inp, {"decisions": want})
|
|
|
|
|
|
rank_case("no_profiles_floor_order", None, None, "fix", False, 0, None)
|
|
rank_case("no_profiles_need_tools", None, None, "fix", True, 0, None)
|
|
rank_case("no_profiles_need_tools_high_context", None, None, "fix", True, 64000, None)
|
|
rank_case("no_profiles_min_context_filters", None, None, "fix", False, 100000, None)
|
|
rank_case("no_profiles_min_context_allows_t4", None, None, "fix", False, 128000, None)
|
|
rank_case("no_profiles_tie_break_tier_then_model",
|
|
[["b-model", "p", "T2", True, 1000, 0.0],
|
|
["a-model", "p", "T10", True, 1000, 0.0],
|
|
["c-model", "p", "T2", True, 1000, 0.0]],
|
|
None, "fix", False, 0, None)
|
|
rank_case("no_profiles_cost_beats_tier",
|
|
[["expensive", "p", "T1", True, 1000, 9.0],
|
|
["cheap", "p", "T9", True, 1000, 1.0]],
|
|
None, "fix", False, 0, None)
|
|
rank_case("weird_tier_sorts_last",
|
|
[["weird", "p", "TX", True, 1000, 0.0],
|
|
["normal", "p", "T3", True, 1000, 0.0]],
|
|
None, "fix", False, 0, None)
|
|
rank_case("empty_registry_is_no_profiles", None, {}, "fix", False, 0, None)
|
|
|
|
FLOOR_PROFILES = [{"provider": "ollama", "model": "qwen2.5:3b", "families": [["fix", (2, 3)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 0.0},
|
|
{"provider": "openai", "model": "mimo-v2.5-pro", "families": [["fix", (0, 2)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 15.0}]
|
|
rank_case("measured_precedence_and_zero_success_skip", None, FLOOR_PROFILES, "fix", False, 0, None)
|
|
rank_case("measured_bypasses_need_tools", None, FLOOR_PROFILES, "fix", True, 0, None)
|
|
rank_case("measured_still_context_limited", None, FLOOR_PROFILES, "fix", False, 100000, None)
|
|
rank_case("measured_other_family_unmeasured", None, FLOOR_PROFILES, "other", False, 0, None)
|
|
rank_case("measured_cost_override",
|
|
None,
|
|
[{"provider": "p", "model": "m", "families": [["fix", (5, 5)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 99.5}],
|
|
"fix", False, 0, None)
|
|
rank_case("zero_n_static_path",
|
|
None,
|
|
[{"provider": "p", "model": "m", "families": [["fix", (3, 0)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 0.0}],
|
|
"fix", False, 0, None)
|
|
rank_case("zero_n_zero_ok_static_path",
|
|
None,
|
|
[{"provider": "p", "model": "m", "families": [["fix", (0, 0)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 0.0}],
|
|
"fix", False, 0, None)
|
|
rank_case("negative_ok_measured",
|
|
None,
|
|
[{"provider": "p", "model": "m", "families": [["fix", (-1, 2)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 2.0}],
|
|
"fix", False, 0, None)
|
|
rank_case("unmeasured_reason_when_profiles_exist", None,
|
|
[{"provider": "openai", "model": "mimo-v2.5-pro", "families": [["fix", (1, 1)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 15.0}],
|
|
"fix", False, 0, None)
|
|
rank_case("unmeasured_need_tools_filtered", None,
|
|
[{"provider": "openai", "model": "mimo-v2.5-pro", "families": [["fix", (1, 1)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 15.0}],
|
|
"fix", True, 0, None)
|
|
rank_case("profiles_path_loaded",
|
|
None, None, "fix", False, 0,
|
|
json.dumps([{"provider": "ollama", "model": "qwen2.5:3b",
|
|
"per_family_success": {"fix": [4, 4]},
|
|
"avg_latency_ms": 1.5, "estimated_cost_usd": 7.5}]))
|
|
rank_case("profiles_path_garbage_is_empty", None, None, "fix", False, 0, "{broken")
|
|
rank_case("custom_candidates_all_branches",
|
|
[["m1", "p", "T1", False, 500, 0.0],
|
|
["m2", "p", "T2", True, 500, 1.0],
|
|
["m3", "p", "T3", True, 200000, 2.0]],
|
|
[{"provider": "p", "model": "m1", "families": [["fix", (1, 1)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 0.5}],
|
|
"fix", True, 1000, None)
|
|
rank_case("tie_cost_measured_vs_static",
|
|
[["static-cheap", "p", "T1", True, 1000, 0.0],
|
|
["measured", "q", "T9", True, 1000, 5.0]],
|
|
[{"provider": "q", "model": "measured", "families": [["fix", (1, 2)]],
|
|
"avg_latency_ms": 0.0, "estimated_cost_usd": 0.0}],
|
|
"fix", False, 0, None)
|
|
rank_case("duplicate_floor_row_ties",
|
|
[["same", "p", "T2", True, 1000, 0.0],
|
|
["same", "p", "T2", True, 1000, 0.0]],
|
|
None, "fix", False, 0, None)
|
|
rank_case("int_cost_candidate",
|
|
[["m", "p", "T1", True, 1000, 5],
|
|
["n", "p", "T1", True, 1000, 5.0]],
|
|
None, "fix", False, 0, None)
|
|
|
|
# ── build_model_router ──────────────────────────────────────────────────────
|
|
for i, (home, explicit) in enumerate([("/rune-home-b", None), ("/rune-home-c", "/x/y.json")]):
|
|
old = os.environ.get("RUNE_HOME")
|
|
os.environ["RUNE_HOME"] = home
|
|
try:
|
|
router = mr.build_model_router(profiles_path=explicit)
|
|
want = profs_json(router._profiles)
|
|
finally:
|
|
if old is None:
|
|
os.environ.pop("RUNE_HOME", None)
|
|
else:
|
|
os.environ["RUNE_HOME"] = old
|
|
case("build_model_router", f"build_{i}", {"home": home, "profiles_path": explicit},
|
|
{"profiles": want})
|
|
|
|
# ── seeded fuzz (seed 2479) ─────────────────────────────────────────────────
|
|
rng = random.Random(2479)
|
|
FAMILIES = ["fix", "refactor", "docs", "research", "ops", "triage"]
|
|
MODELS = ["q:3b", "q3:14b", "mimo-v2.5-pro", "cloud-x", "q"]
|
|
PROVIDERS = ["ollama", "openai", "cloud"]
|
|
SAFE_FLOATS = [0.0, 0.5, 1.5, 2.0, 12.5, 15.0, 99.5, 123.25, 999.5, 4.75]
|
|
|
|
|
|
def garbage_variant(row):
|
|
r = dict(row)
|
|
k = rng.choice(["stats_bool", "stats_float", "latency_str", "latency_null",
|
|
"provider_int", "family_int", "drop_stats"])
|
|
if k == "stats_bool":
|
|
r["per_family_success"] = {"g": [True, 2]}
|
|
elif k == "stats_float":
|
|
r["per_family_success"] = {"g": [1.5, 2]}
|
|
elif k == "latency_str":
|
|
r["avg_latency_ms"] = "abc"
|
|
elif k == "latency_null":
|
|
r["avg_latency_ms"] = None
|
|
elif k == "provider_int":
|
|
r["provider"] = 9
|
|
elif k == "family_int":
|
|
r["per_family_success"] = {3: [1, 2]}
|
|
else:
|
|
r.pop("per_family_success", None)
|
|
return r
|
|
|
|
|
|
for i in range(100):
|
|
rows = []
|
|
for _ in range(rng.randint(0, 4)):
|
|
row = {"provider": rng.choice(PROVIDERS),
|
|
"model": rng.choice(MODELS),
|
|
"per_family_success": {f: [rng.randint(0, 5), rng.randint(0, 6)] for f in rng.sample(FAMILIES, rng.randint(1, 3))},
|
|
"avg_latency_ms": rng.choice(SAFE_FLOATS),
|
|
"estimated_cost_usd": rng.choice(SAFE_FLOATS)}
|
|
if rng.random() < 0.2:
|
|
row = garbage_variant(row)
|
|
if rng.random() < 0.1:
|
|
row.pop("avg_latency_ms")
|
|
rows.append(row)
|
|
content = json.dumps(rows)
|
|
if rng.random() < 0.07:
|
|
cut = rng.randint(1, len(content) - 1)
|
|
content = content[:cut]
|
|
load_case_fuzz = f"fuzz_load_{i}"
|
|
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
|
|
f.write(content)
|
|
path = f.name
|
|
reg = mr.load_profiles(path)
|
|
os.unlink(path)
|
|
case("load_profiles", load_case_fuzz, {"file": content}, {"profiles": profs_json(reg)})
|
|
|
|
for i in range(100):
|
|
cands = []
|
|
for _ in range(rng.randint(1, 5)):
|
|
tier = rng.choice(["T1", "T2", "T3", "T4", "T9", "T10", "TX", "T100"])
|
|
cost = rng.choice([0.0, 0.5, 1.0, 2.5, 5, 15.0, 99.5])
|
|
cands.append([rng.choice(MODELS), rng.choice(PROVIDERS), tier,
|
|
rng.random() < 0.6, rng.choice([1024, 32768, 128000, 500]),
|
|
cost])
|
|
profiles = None
|
|
if rng.random() < 0.75:
|
|
profiles = []
|
|
for _ in range(rng.randint(1, 4)):
|
|
fams = {}
|
|
for f in rng.sample(FAMILIES, rng.randint(1, 3)):
|
|
fams[f] = [rng.choice([0, 0, 1, 2, 3, -1]), rng.randint(0, 5)]
|
|
profiles.append({"provider": rng.choice(PROVIDERS), "model": rng.choice(MODELS),
|
|
"families": [[f, [v[0], v[1]]] for f, v in fams.items()],
|
|
"avg_latency_ms": rng.choice(SAFE_FLOATS),
|
|
"estimated_cost_usd": rng.choice(SAFE_FLOATS)})
|
|
rank_case(f"fuzz_rank_{i}", cands, profiles,
|
|
rng.choice(FAMILIES + ["nomatch"]), rng.random() < 0.4,
|
|
rng.choice([0, 0, 1000, 32768, 64000, 128000, 200000]), None)
|
|
|
|
for i in range(100):
|
|
start = None
|
|
if rng.random() < 0.7:
|
|
rows = []
|
|
for _ in range(rng.randint(0, 3)):
|
|
rows.append({"provider": rng.choice(PROVIDERS), "model": rng.choice(MODELS),
|
|
"per_family_success": {f: [rng.randint(0, 3), rng.randint(1, 5)]
|
|
for f in rng.sample(FAMILIES, rng.randint(1, 2))},
|
|
"avg_latency_ms": rng.choice(SAFE_FLOATS),
|
|
"estimated_cost_usd": rng.choice(SAFE_FLOATS)})
|
|
start = json.dumps(rows)
|
|
if rng.random() < 0.08:
|
|
start = "{broken"
|
|
traces = []
|
|
for _ in range(rng.randint(0, 6)):
|
|
t = {"provider": rng.choice(PROVIDERS + ["", 9, None]),
|
|
"model": rng.choice(MODELS + ["", None]),
|
|
"success": rng.choice([True, False, 1, "true", None]),
|
|
"task_family": rng.choice(FAMILIES + ["", None, 0])}
|
|
if rng.random() < 0.15:
|
|
t.pop("task_family")
|
|
if rng.random() < 0.1:
|
|
t.pop("success")
|
|
traces.append(t)
|
|
update_case(f"fuzz_update_{i}", start, traces, task_family=rng.choice(FAMILIES + [""]))
|
|
|
|
# ── write ───────────────────────────────────────────────────────────────────
|
|
out_dir = os.path.join("src", "go", "model_routing")
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
with open(os.path.join(out_dir, "truth.json"), "w", encoding="utf-8") as f:
|
|
json.dump(cases, f, indent=1, ensure_ascii=True)
|
|
print(f"FROZEN {len(cases)} cases -> src/go/model_routing/truth.json")
|
|
from collections import Counter
|
|
print(Counter(c["fn"] for c in cases))
|