conductor/rune
1
2 Fork 1
Code Issues 38 Pull Requests 32 Actions 10 Packages Projects Releases 564 Wiki Activity
Files
2eb3227ad0ec57fb259d12f754a183dfafb94de2
rune/scripts/gen_context_broker_truth.py
T

384 lines
20 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""Freeze the oracle truth for the GO gap-close port of
src/rune/context_broker.py (dichotomy wave #2479, GO, factory-gap-ctxbroker).
Every NEW function ported by the gap-close, every branch, against the LIVE
oracle pinned to the Python cores (RUNE_COGNITION_CORE=python — the _py_*
oracles ARE the parity floor being frozen). Deep-copied inputs: the oracle
mutates message lists in place (compact_history archive path), so each row
re-freezes the input verbatim.
Ops frozen (fn = the Go-side dispatcher key):
_py_default_estimator est (text)
_cc_core cc_core (env RUNE_COGNITION_CORE) -> bool available
_py_default_budget budget (window)
_py_default_memory_budget membud (window, incl. None/str/bool coercion)
_py_memory_disabled_value memoff (raw string)
reset_discovered_windows / register_window / default_window
windows (sequence of ops -> final registry view)
summarize_history summarize (transport-failure branch: ValueError)
compact_history compact (messages, window, summarizer, session)
rolling_summary_state rstate (key -> state | null)
_through_id through (messages, through_count)
store_rolling_summary / evict_rolling rstore (op sequence -> state + return)
"""
from __future__ import annotations
import copy
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
os.environ["RUNE_COGNITION_CORE"] = "python" # pin the Python parity floor
from rune import context_broker as cb # noqa: E402
rows: list[dict] = []
def row(name, fn, inp, out):
rows.append({"name": name, "fn": fn, "input": inp, "want": out})
# ── _py_default_estimator: every branch ────────────────────────────────────
row("est:empty", "est", {"text": ""}, cb._py_default_estimator(""))
row("est:none-coerced-empty", "est", {"text": None}, cb._py_default_estimator(None))
row("est:short", "est", {"text": "hello"}, cb._py_default_estimator("hello"))
row("est:exact-8", "est", {"text": "12345678"}, cb._py_default_estimator("12345678"))
row("est:9-floor", "est", {"text": "123456789"}, cb._py_default_estimator("123456789"))
row("est:unicode-rune-len", "est", {"text": "héllo wörld"}, cb._py_default_estimator("héllo wörld"))
# ── _cc_core: python-pin + auto-without-build ───────────────────────────────
from rune import cc_loader # noqa: E402
_LOADER_AVAIL = bool(cc_loader.available())
row("cc_core:python-pin-returns-none", "cc_core",
{"env": {"RUNE_COGNITION_CORE": "python"}, "loader_available": _LOADER_AVAIL},
None if cb._cc_core() is None else "CORE")
del os.environ["RUNE_COGNITION_CORE"]
row("cc_core:auto-fallback", "cc_core", {"env": {}, "loader_available": _LOADER_AVAIL},
None if cb._cc_core() is None else "CORE")
# ── _py_default_budget: every branch ────────────────────────────────────────
row("budget:0-floor", "budget", {"window": 0}, cb._py_default_budget(0))
row("budget:1-floor", "budget", {"window": 1}, cb._py_default_budget(1))
row("budget:100-floor", "budget", {"window": 100}, cb._py_default_budget(100))
row("budget:4096", "budget", {"window": 4096}, cb._py_default_budget(4096))
row("budget:8192", "budget", {"window": 8192}, cb._py_default_budget(8192))
row("budget:131072", "budget", {"window": 131072}, cb._py_default_budget(131072))
row("budget:quarter-exact-8192", "budget", {"window": 8192}, cb._py_default_budget(8192))
# ── _py_default_memory_budget: every branch incl. coercion ─────────────────
row("membud:0-cap", "membud", {"window": 0}, cb._py_default_memory_budget(0))
row("membud:none-cap", "membud", {"window": None}, cb._py_default_memory_budget(None))
row("membud:neg-cap", "membud", {"window": -5}, cb._py_default_memory_budget(-5))
row("membud:8-eq-1", "membud", {"window": 8}, cb._py_default_memory_budget(8))
row("membud:2400", "membud", {"window": 2400}, cb._py_default_memory_budget(2400))
row("membud:9600-cap", "membud", {"window": 9600}, cb._py_default_memory_budget(9600))
row("membud:96000-cap", "membud", {"window": 96000}, cb._py_default_memory_budget(96000))
# bool/float coerce through the public fn's int(window) path when a Rust core
# is present; on the Python floor they hit _py_default_memory_budget raw.
row("membud:bool-true", "membud", {"window": True}, cb._py_default_memory_budget(True))
row("membud:float", "membud", {"window": 2400.9}, cb._py_default_memory_budget(2400.9))
# ── _py_memory_disabled_value: every branch ────────────────────────────────
for v in ("", "0", "false", "no", "off", "1", "true", "yes", "on", "OFF",
" False ", "anything", " ", "off\n"):
row(f"memoff:{v!r}", "memoff", {"value": v}, cb._py_memory_disabled_value(v))
# ── window registry: reset/register/default with NO live discovery ─────────
# default_window's live-discovery import is exercised in the fallback branch
# only when model_id is falsy (registry miss + import of local_tool_runtime
# would attempt a real HTTP call otherwise — the Go seam stubs discovery;
# here we freeze: falsy id, registry hit, registry miss w/ capped discovery).
cb.reset_discovered_windows()
row("windows:reset-then-missing-none", "windows",
{"ops": [{"op": "default_window", "model_id": None}]},
{"last": cb.default_window(None)})
row("windows:none-empty-str", "windows",
{"ops": [{"op": "default_window", "model_id": ""}]},
{"last": cb.default_window("")})
row("windows:register-hit", "windows",
{"ops": [{"op": "register_window", "model_id": "m1", "window": 32768},
{"op": "default_window", "model_id": "m1"}]},
{"last": (cb.reset_discovered_windows(), cb.register_window("m1", 32768),
cb.default_window("m1"))[2]})
row("windows:register-overwrite", "windows",
{"ops": [{"op": "register_window", "model_id": "m1", "window": 8192},
{"op": "register_window", "model_id": "m1", "window": 65536},
{"op": "default_window", "model_id": "m1"}]},
{"last": (cb.reset_discovered_windows(), cb.register_window("m1", 8192),
cb.register_window("m1", 65536), cb.default_window("m1"))[3]})
row("windows:reset-clears", "windows",
{"ops": [{"op": "register_window", "model_id": "m1", "window": 8192},
{"op": "reset_discovered_windows"},
{"op": "default_window", "model_id": "m1"}]},
{"last": (cb.reset_discovered_windows(), cb.register_window("m1", 8192),
cb.reset_discovered_windows(), cb.default_window("m1"))[3]})
row("windows:register-str-window-coerced", "windows",
{"ops": [{"op": "register_window", "model_id": "m1", "window": "28672"},
{"op": "default_window", "model_id": "m1"}]},
{"last": (cb.reset_discovered_windows(), cb.register_window("m1", "28672"),
cb.default_window("m1"))[2]})
# discovery-failure branch: unknown model -> the live oracle tries the
# transport; in this offline env that raises inside context_length ->
# record_failure + fallback 8192 (never cached). Freeze the returned value.
cb.reset_discovered_windows()
row("windows:miss-discover-fallback-8192", "windows",
{"ops": [{"op": "default_window", "model_id": "definitely-not-a-model:offline"}]},
{"last": cb.default_window("definitely-not-a-model:offline"),
"note": "live oracle: T1 context_discovery + FALLBACK_WINDOW, not cached"})
# ── summarize_history: transport-failure branch (offline) ──────────────────
try:
cb.summarize_history([{"role": "user", "content": "hi"}], model="offline-no-such-model")
row("summarize:unexpected-success", "summarize", {"messages": []}, "UNEXPECTED-SUCCESS")
except ValueError as exc:
row("summarize:transport-fail-valueerror", "summarize",
{"messages": [{"role": "user", "content": "hi"}]},
{"error": "ValueError", "detail": str(exc)[:120]})
except Exception as exc: # noqa: BLE001 — freeze whatever the live env does
row("summarize:transport-fail-other", "summarize",
{"messages": [{"role": "user", "content": "hi"}]},
{"error": type(exc).__name__, "detail": str(exc)[:120]})
# ── _through_id: every branch ───────────────────────────────────────────────
msgs = [{"role": "user", "content": "first"},
{"role": "assistant", "content": "reply"},
{"role": "user", "content": "second"}]
row("through:empty-list", "through", {"messages": [], "through": 0}, cb._through_id([], 0))
row("through:neg", "through", {"messages": msgs, "through": -1}, cb._through_id(msgs, -1))
row("through:at-end", "through", {"messages": msgs, "through": 3}, cb._through_id(msgs, 3))
row("through:beyond-end", "through", {"messages": msgs, "through": 99}, cb._through_id(msgs, 99))
row("through:anchor-0", "through", {"messages": msgs, "through": 0}, cb._through_id(msgs, 0))
row("through:anchor-1", "through", {"messages": msgs, "through": 1}, cb._through_id(msgs, 1))
row("through:content-hash-stable", "through",
{"messages": msgs, "through": 2},
(cb._through_id(msgs, 2), cb._through_id(copy.deepcopy(msgs), 2),
cb._through_id([{"content": "second", "role": "user"}], 0))[0],
)
# key-order independence (sort_keys=True)
row("through:key-order-independent", "through",
{"messages": [{"role": "user", "content": "x"}, {"content": "x", "role": "user"}],
"through": 1},
cb._through_id([{"role": "user", "content": "x"}, {"content": "x", "role": "user"}], 1))
# ── rolling summary store: state/store/evict (self-contained rows) ──────────
cb._ROLLING.clear()
row("rstate:miss", "rstate", {"ops": [{"op": "read", "source": "operator", "sid": "s1"}]},
cb.rolling_summary_state(("operator", "s1")))
cb.store_rolling_summary(("operator", "s1"), "SUM", 7, "abcdef0123456789")
row("rstate:hit-copy", "rstate",
{"ops": [{"op": "store", "source": "operator", "sid": "s1", "summary": "SUM",
"through": 7, "tid": "abcdef0123456789"},
{"op": "read", "source": "operator", "sid": "s1"}]},
cb.rolling_summary_state(("operator", "s1")))
row("rstore:store-then-through-id", "rstore",
{"msgs": msgs},
(cb.store_rolling_summary(("operator", "s2"), "S2", 1, cb._through_id(msgs, 1)),
cb.rolling_summary_state(("operator", "s2")))[1])
# ── compact_history: every branch (injected summarizer = deterministic) ────
def fresh(seed: str, n_pairs: int) -> list[dict]:
out = []
for i in range(n_pairs):
out.append({"role": "user", "content": f"{seed} user {i} " + "x" * (i * 40)})
out.append({"role": "assistant", "content": f"{seed} assistant {i} " + "y" * (i * 40)})
return out
def sumr(messages):
return "SUMMARY<" + "|".join(m.get("content", "")[:10] for m in messages) + ">"
def compact_row(name, messages, window, *, summarizer="sum", session="", prior=None,
archive=False):
"""Freeze one compact_history call; self-contained (prior state inline)."""
cb._ROLLING.clear()
if prior:
cb.store_rolling_summary(("operator", session) if session else ("operator",),
prior["summary"], prior["through_count"],
prior.get("through_id", ""))
kw = dict(summarizer=sumr if summarizer != "boom" else boom)
arch = None
if archive:
# deterministic RELATIVE fixture path (chdir'd to repo root): the
# label is embedded in the summary block, so it must be portable
os.chdir(ROOT)
arch = Path("src/go/context-broker/.fixtures/archive.jsonl")
arch.parent.mkdir(parents=True, exist_ok=True)
inp = {"messages": copy.deepcopy(messages), "window": window,
"summarizer": summarizer}
if session:
inp["session"] = session
if prior:
inp["prior"] = prior
if archive:
inp["archive"] = "FIXTURE"
inp["archive_path"] = str(arch)
compacted, ev = cb.compact_history(copy.deepcopy(messages), window,
archive_path=arch,
summarizer=sumr if summarizer != "boom" else boom,
session_id=session)
out = {"event": ev}
if messages is not None:
out["compacted"] = compacted
if archive:
body = arch.read_text() if arch.exists() else ""
out["archive_lines"] = body.count("\n")
out["archived_msg_count"] = len(json.loads(body.splitlines()[0])["messages"]) if body else 0
out["archived_msgs"] = json.loads(body.splitlines()[0])["messages"] if body else []
arch.unlink(missing_ok=True)
row(name, "compact", inp, out)
def boom(messages):
raise RuntimeError("summarizer down")
# no messages -> early return, event untriggered
compact_row("compact:empty-messages", [], 8192)
# <= KEEP_TURNS user turns -> early return
compact_row("compact:at-keep-turns-no-trigger", fresh("a", 5), 8192)
compact_row("compact:below-keep-turns-no-trigger", fresh("a", 4), 8192)
# > keep turns, no session, summarizer success
compact_row("compact:trigger-no-session", fresh("b", 7), 8192)
# with session: rolling persistence across TWO compactions (C1: only new)
cb._ROLLING.clear()
m3 = fresh("c", 8)
c1, ev1 = cb.compact_history(copy.deepcopy(m3), 8192, summarizer=sumr,
session_id="s3")
prior3 = cb.rolling_summary_state(("operator", "s3"))
row("compact:session-first", "compact",
{"messages": m3, "window": 8192, "session": "s3", "summarizer": "sum"},
{"event": ev1, "compacted": c1})
# second compaction: window rotated under the stored count (#141 rematch)
m3b = m3[:2] + m3[-10:]
compact_row("compact:session-rotation-rematch", m3b, 8192, session="s3",
prior=prior3)
# prior through beyond len(older) — history shrank, re-summarize from scratch
compact_row("compact:session-through-beyond-shrink", fresh("d", 7), 8192,
session="s4", prior={"summary": "PRIOR-SUM", "through_count": 999,
"through_id": ""})
# nothing new beyond the rolled summary: reuse it verbatim
cb._ROLLING.clear()
m9 = fresh("g", 7)
_, ev9_pre = cb.compact_history(copy.deepcopy(m9), 8192, summarizer=sumr,
session_id="s9")
prior9 = cb.rolling_summary_state(("operator", "s9"))
# call again with IDENTICAL messages: older[6:] is empty -> reuse verbatim
compact_row("compact:session-nothing-new-reuse", m9, 8192, session="s9",
prior=prior9)
# summarizer failure -> C2 degrade to tail
compact_row("compact:summarizer-fail-degrade-tail", fresh("e", 7), 8192,
summarizer="boom")
# summarizer failure with tiny window (tail budget floor: always keep >=1)
m6 = [{"role": "user", "content": "u" * 9000}, {"role": "assistant", "content": "a" * 9000},
{"role": "user", "content": "u2" * 9000}, {"role": "assistant", "content": "a2" * 9000},
{"role": "user", "content": "u3" * 9000}, {"role": "assistant", "content": "a3" * 9000},
{"role": "user", "content": "u4" * 9000}]
compact_row("compact:fail-tiny-window-tail-first-only", m6, 10, summarizer="boom")
# archive write: success path — event.archived True, label = the path itself
compact_row("compact:archive-success", fresh("f", 6), 8192, session="s7",
archive=True)
# default_estimator used inside event tokens (python pin): tokens_before math
compact_row("compact:token-math-estimator",
[{"role": "user", "content": "12345678"}, {"role": "assistant", "content": "1234"}],
8192)
# ── ContextBroker kernel: to_dict / collect / admit / render ────────────────
from rune.context_broker import ContributionBlock, ContextBroker, ContextOverflowError # noqa: E402
# to_dict
blk = ContributionBlock("memory", 5, "hello world", 2, "recall", evictable=True)
row("to_dict:block", "to_dict", {}, blk.to_dict())
blk2 = ContributionBlock("system", 100, "sys", 0, "persona", evictable=False)
row("to_dict:mandatory", "to_dict", {}, blk2.to_dict())
# collect: happy, empty-content skip, failure → T1 degrade, malformed → T1
def make_broker():
b = ContextBroker()
b.register_contributor("ok", lambda task, session: ContributionBlock("tools", 9, "tooltext", 2, "t"), 1)
b.register_contributor("empty", lambda task, session: ContributionBlock("x", 1, "", 0, "s"), 1)
b.register_contributor("boom", lambda task, session: (_ for _ in ()).throw(RuntimeError("cb down")), 1)
b.register_contributor("malformed", lambda task, session: ContributionBlock("y", 1, 12345, 0, "s"), 1)
b.register_contributor("list-ok", lambda task, session: ContributionBlock("history", 3, [{"role": "user", "content": "hi"}], 1, "h"), 2)
b.register_contributor("none", lambda task, session: None, 1)
return b
b = make_broker()
degraded: list[str] = []
collected = b._collect({}, {}, degraded)
row("collect:degraded-and-ordered", "collect", {},
{"names": [name for name, _blk, _b in collected],
"degraded": degraded,
"failed_events": sorted(getattr(b, "_last_failures", []))})
# admit: priority desc, recency tie, memory sub-budget, overflow
def blocks(*specs):
out = []
for name, pri, tokens, evictable in specs:
out.append(ContributionBlock(name, pri, "c" * tokens, tokens, f"src-{name}", evictable=evictable))
return out
bk = ContextBroker()
admitted, adm = bk._admit([(b["name"], bl, 0) for b, bl in zip(
[{"name": "system"}, {"name": "memory"}, {"name": "history"}, {"name": "user"}],
blocks(("system", 100, 500, False), ("memory", 5, 300, True),
("history", 7, 900, True), ("user", 100, 100, False)))], 1000)
row("admit:priority-recency-budget", "admit", {},
{"admitted": [bl.name for bl in admitted],
"blocks": adm["blocks"], "admitted_total": adm["admitted_total"],
"dropped": [d["name"] for d in adm["dropped"]],
"memory_budget": adm["memory_budget"], "memory_used": adm["memory_used"]})
# memory sub-budget: two memory blocks, second exceeds sub-budget
admitted2, adm2 = bk._admit([("m1", blocks(("m1", 9, 700, True))[0], 0),
("m2", blocks(("m2", 8, 700, True))[0], 1)], 4000, memory_budget=1200)
row("admit:memory-subbudget-drop", "admit", {},
{"admitted": [bl.name for bl in admitted2],
"dropped": [d["name"] for d in adm2["dropped"]],
"memory_used": adm2["memory_used"],
"reasons": [d["reason"] for d in adm2["dropped"]]})
# mandatory overflow → raise
try:
bk._admit([("user", blocks(("user", 100, 5000, False))[0], 0)], 1000)
row("admit:mandatory-overflow-raises", "admit", {}, "NO-RAISE")
except ContextOverflowError as exc:
row("admit:mandatory-overflow-raises", "admit", {},
{"error": "ContextOverflowError", "detail": str(exc)})
# render: system folding, history pass-through, user capture
capsule = bk._render(blocks(("persona", 100, 7, False), ("tools", 90, 5, False),
("history", 70, 4, False), ("user", 100, 2, False)))
row("render:order-and-fold", "render", {},
{"token_count": capsule.token_count,
"messages": capsule.messages,
"system_preamble": capsule.system_preamble,
"user_content": capsule.user_content,
"blocks": [bl.name for bl in capsule.blocks]})
print(f"froze {len(rows)} rows total")
doc = {
"frozen_ts": "2026-10-12T00:00:00Z",
"generator": "scripts/gen_context_broker_truth.py — live oracle src/rune/context_broker.py, RUNE_COGNITION_CORE=python pinned",
"rows": rows,
}
out = ROOT / "src/go/context-broker/truth.json"
out.write_text(json.dumps(doc, indent=1, sort_keys=True) + "\n")
Reference in new issue View Git Blame Copy Permalink