181 lines
6.8 KiB
Python
181 lines
6.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Live parity check: the Go context-broker binary vs the LIVE Python oracle
|
|
(RUNE_COGNITION_CORE=python pinned AND delegation auto), fuzz + edge cases.
|
|
Complements the frozen battery (truth.json) with generated inputs."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import random
|
|
import string
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
|
|
BIN = Path("/tmp/cb-live-parity")
|
|
|
|
|
|
def build():
|
|
subprocess.run(["go", "build", "-o", str(BIN), "."],
|
|
cwd=ROOT / "src/go/context-broker", check=True)
|
|
|
|
|
|
def go_eval(payload: dict) -> dict:
|
|
proc = subprocess.run([str(BIN), "eval"], input=json.dumps(payload),
|
|
capture_output=True, text=True)
|
|
if proc.returncode != 0:
|
|
raise RuntimeError(f"go eval failed: {proc.stderr}")
|
|
return json.loads(proc.stdout)
|
|
|
|
|
|
fails = 0
|
|
checked = 0
|
|
|
|
|
|
def check(name, got, want):
|
|
global fails, checked
|
|
checked += 1
|
|
if got != want:
|
|
fails += 1
|
|
print(f"DIVERGE {name}: go={got!r} py={want!r}")
|
|
|
|
|
|
def est_case(text):
|
|
check(f"est[{text!r:.40}]", go_eval({"op": "est", "text": text}),
|
|
py._py_default_estimator(text))
|
|
|
|
|
|
def budget_case(w):
|
|
check(f"budget[{w}]", go_eval({"op": "budget", "window": w}),
|
|
py._py_default_budget(w))
|
|
|
|
|
|
def membud_case(w):
|
|
check(f"membud[{w}]", go_eval({"op": "membud", "window": w}),
|
|
py._py_default_memory_budget(w))
|
|
|
|
|
|
def memoff_case(v):
|
|
check(f"memoff[{v!r}]", go_eval({"op": "memoff", "value": v}),
|
|
py._py_memory_disabled_value(v))
|
|
|
|
|
|
def through_case(messages, through):
|
|
want = py._through_id(json.loads(json.dumps(messages)), through)
|
|
got = go_eval({"op": "through", "messages": messages, "through": through})
|
|
check(f"through[{through}/{len(messages)}]", got, want)
|
|
|
|
|
|
rng = random.Random(2479)
|
|
|
|
os.environ["RUNE_COGNITION_CORE"] = "python"
|
|
from rune import context_broker as py # noqa: E402
|
|
|
|
# fuzz: estimator
|
|
for _ in range(200):
|
|
n = rng.randint(0, 300)
|
|
est_case("".join(rng.choice(string.printable[:94]) for _ in range(n)))
|
|
est_case("héllo wörld ünïcode ✓ 日本語テキスト")
|
|
est_case("🎉 emoji surrogate pair")
|
|
|
|
# fuzz: budgets (incl. boundaries around max(2048, w/4) crossover)
|
|
for w in [0, 1, 2559, 2560, 2561, 8191, 8192, 8193, 10000, 65536, 131072,
|
|
200000, 10**9] + [rng.randint(0, 200000) for _ in range(100)]:
|
|
budget_case(w)
|
|
membud_case(w)
|
|
|
|
# fuzz: memory-off raw values
|
|
for v in ["", "0", "00", "false", "FALSE", "False", "no", "No", "off", "OFF",
|
|
"off ", " off", "1", "true", "YES", "on", "anything", "\t", "\n",
|
|
"0.0", "nil", "None", "null"]:
|
|
memoff_case(v)
|
|
|
|
# fuzz: through_id over random message shapes
|
|
for _ in range(100):
|
|
msgs = []
|
|
for _ in range(rng.randint(1, 8)):
|
|
m = {"role": rng.choice(["user", "assistant", "system"]),
|
|
"content": "".join(rng.choice(string.printable[:70]) for _ in range(rng.randint(0, 40)))}
|
|
if rng.random() < 0.3:
|
|
m["extra"] = {"n": rng.randint(0, 5), "flag": rng.random() < 0.5}
|
|
msgs.append(m)
|
|
through_case(msgs, rng.randint(-1, len(msgs) + 1))
|
|
|
|
# unicode through_id (ensure_ascii path)
|
|
uni = [{"role": "user", "content": "日本語 テキスト 🎉 emoji"},
|
|
{"role": "assistant", "content": "héllo wörld"}]
|
|
through_case(uni, 1)
|
|
through_case(uni, 0)
|
|
|
|
# fuzz: admission kernel (priority desc, recency tie, budget, memory cap)
|
|
NAMES = ["system", "user", "history", "memory", "persona_memories", "tools", "notes"]
|
|
for case_i in range(150):
|
|
entries = []
|
|
for j in range(rng.randint(1, 7)):
|
|
name = rng.choice(NAMES)
|
|
entries.append({
|
|
"name": name,
|
|
"priority": rng.randint(0, 100),
|
|
"tokens": rng.randint(1, 900),
|
|
"source": f"src-{j}",
|
|
"evictable": name not in ("system", "user"),
|
|
})
|
|
budget = rng.choice([500, 1000, 2000, 4000])
|
|
mem_budget = rng.choice([0, 300, 1200])
|
|
got = go_eval({"op": "admit", "entries": entries, "budget": budget,
|
|
"memory_budget": mem_budget})
|
|
bk = py.ContextBroker()
|
|
py_entries = [(e["name"], py.ContributionBlock(e["name"], e["priority"],
|
|
"c" * e["tokens"], e["tokens"],
|
|
e["source"], evictable=e["evictable"]), j)
|
|
for j, e in enumerate(entries)]
|
|
try:
|
|
_adm, adm = bk._admit(py_entries, budget, memory_budget=mem_budget)
|
|
want = {"admitted": [bl.name for bl in _adm],
|
|
"blocks": adm["blocks"], "admitted_total": adm["admitted_total"],
|
|
"dropped": [d["name"] for d in adm["dropped"]],
|
|
"memory_budget": adm["memory_budget"], "memory_used": adm["memory_used"]}
|
|
check(f"admit[{case_i}]", got, json.loads(json.dumps(want, sort_keys=True)))
|
|
except py.ContextOverflowError as exc:
|
|
# Go wraps the oracle's message with the wrapped sentinel; the oracle
|
|
# text must be a verbatim prefix of the Go detail.
|
|
ok = (isinstance(got, dict) and got.get("error") == "ContextOverflowError"
|
|
and got.get("detail", "").startswith(str(exc)))
|
|
check(f"admit[{case_i}]", "ContextOverflowError" if ok else got,
|
|
"ContextOverflowError")
|
|
|
|
# fuzz: render (system folding, history pass-through, user capture)
|
|
for case_i in range(100):
|
|
blocks = []
|
|
for j in range(rng.randint(1, 6)):
|
|
name = rng.choice(NAMES)
|
|
if rng.random() < 0.5:
|
|
content = "".join(rng.choice(string.ascii_letters) for _ in range(rng.randint(1, 30)))
|
|
else:
|
|
content = [{"role": rng.choice(["user", "assistant"]),
|
|
"content": "".join(rng.choice(string.ascii_letters) for _ in range(rng.randint(1, 20)))}
|
|
for _ in range(rng.randint(1, 3))]
|
|
blocks.append({"name": name, "priority": rng.randint(0, 100),
|
|
"tokens": 0, "source": f"src-{j}", "content": content})
|
|
got = go_eval({"op": "render", "blocks": blocks})
|
|
py_blocks = []
|
|
for b in blocks:
|
|
content = b["content"]
|
|
if isinstance(content, list):
|
|
content = [dict(m) for m in content]
|
|
py_blocks.append(py.ContributionBlock(b["name"], b["priority"], content,
|
|
b["tokens"], b["source"]))
|
|
caps = py.ContextBroker()._render(py_blocks)
|
|
want = {"token_count": caps.token_count, "messages": caps.messages,
|
|
"system_preamble": caps.system_preamble, "user_content": caps.user_content,
|
|
"blocks": [bl.name for bl in caps.blocks]}
|
|
check(f"render[{case_i}]", got, json.loads(json.dumps(want)))
|
|
|
|
print(f"live parity: {checked - fails}/{checked} cases match "
|
|
f"(RUNE_COGNITION_CORE=python pinned)")
|
|
sys.exit(1 if fails else 0)
|