Nick
a858ab1392
audit 3/8 PARTIAL -> 8/8 FULL (GO side). Ported to_dict (CognitiveJob/CognitiveResult via the real constructors), the full execute surface (no-profile early return with attempts=1 and zero latency, T0-deterministic vs model dispatch), _execute_model (OO OLLAMA payload json.dumps byte-parity: ', '/' : ' separators, insertion order, ensure_ascii + surrogate pairs, rune-safe [:2000] response slice, crash-parity envelopes for unknown-url-type + connection-refused), _ctx_budget + vendored ResolveCtxBudget (GO SIBLING ISOLATION law), and test_executor assert-parity (Python True/False literals, crash exit 1 after the Coding line).
Frozen truth: 27-case envelope via the shared freeze_truth.py + scripts/te_freeze_seams rig (time.time pinned -> latency 0.0 mirrored by the TE_LATENCY_MS seam; OLLAMA echo server reflects the request body so payload bytes ride the truth rows; ModelRegistry({}) falsiness -> DEFAULT_PROFILES, no-profile rows via the T9 sentinel). Verified byte-identical twice. Battery 11 tests green; CI replay binary PARITY OK 27/27; live parity sweep 31/31 (25 op families + 6 CLI incl. test / execute / usage); gofmt/vet/build clean; python smoke both pins (no loader seam - vacuous pin, PP precedent).
Evidence: verification/te-*.exit/*.exit files + te-summary.md (logs local, * log gitignored).
177 lines
10 KiB
Python
177 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""gen_tiered_executor_truth.py — freeze the tiered_executor oracle truth
|
|
(dichotomy gap-close, wave #2479) through the SHARED harness
|
|
(scripts/freeze_truth.py) under the scripts/te_freeze_seams rig.
|
|
|
|
Surfaces frozen (the 5 audit-missing fns + the already-covered surface,
|
|
so the battery replays ONE envelope):
|
|
* CognitiveJob.to_dict / CognitiveResult.to_dict — routed through the
|
|
REAL constructors (defaults intact) by the rig's _te_* seams
|
|
* TieredExecutor.execute — deterministic family paths (rune-safe
|
|
[:100] truncation rows, unicode + emoji), no-profile early returns
|
|
via subset registries built by the rig, model paths against the
|
|
rig's OLLAMA echo server (payload bytes pinned via the echo round-
|
|
trip), bad-scheme + connection-refused crash-parity rows
|
|
* _ctx_budget — direct rows over the vendored resolve_context_budget
|
|
surface (local/cloud/unknown model ids)
|
|
* test_executor — captured assert-safe ({"returned": null})
|
|
|
|
latency_ms is exactly 0.0 in every row: the rig freezes time.time and the
|
|
Go battery pins the TE_LATENCY_MS seam to the same epoch law.
|
|
|
|
Usage: PYTHONPATH=scripts/te_freeze_seams REPO_ROOT=$PWD python3 scripts/gen_tiered_executor_truth.py
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO / "scripts"))
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
|
|
from freeze_truth import freeze, write_envelope # noqa: E402
|
|
|
|
FROZEN_NOW = "2026-10-14T10:00:00+00:00"
|
|
OUT = REPO / "src" / "go" / "tiered-executor" / "truth.json"
|
|
|
|
# A real T3 profile dict (subset registries build ModelProfile(**d) from it)
|
|
T3_PROFILE = {
|
|
"id": "qwen3.6:27b", "provider": "ollama", "tier": "T3", "local": True,
|
|
"context_window": 32768, "supports_tools": False,
|
|
"supports_structured_output": False, "supports_reasoning_field": False,
|
|
"estimated_latency": "high", "estimated_cost": "0", "enabled": True,
|
|
}
|
|
|
|
# objective mixing BMP escapes, an astral surrogate pair, quotes, backslash,
|
|
# a tab and a newline — payload must exceed 2000 chars to exercise [:2000]
|
|
WIDE = ("héllo wörld " * 80) + ('日本語 "quoted" \\backslash\t\n') + ("🚀" * 30)
|
|
|
|
# ModelRegistry({}) is falsy -> falls back to DEFAULT_PROFILES, so a
|
|
# "no profiles" registry must carry a sentinel profile under a tier no
|
|
# case selects: every get_profile(<real tier>) then misses -> None.
|
|
EMPTY_REG = {"T9": {**T3_PROFILE, "id": "none", "tier": "T9"}}
|
|
|
|
LONG_ASCII_105 = "x" * 105
|
|
|
|
BUDGET_T0 = {"start_tier": "T0", "max_tier": "T0"}
|
|
BUDGET_T1 = {"start_tier": "T1", "max_tier": "T2"}
|
|
BUDGET_T2 = {"start_tier": "T2", "max_tier": "T4"}
|
|
BUDGET_T3 = {"start_tier": "T3", "max_tier": "T4"}
|
|
BUDGET_T4 = {"start_tier": "T4", "max_tier": "T4"}
|
|
|
|
CASES = [
|
|
# ── CognitiveJob.to_dict via the real constructor ────────────────────────
|
|
("_te_job_to_dict", ({"job_id": "m1", "task_family": "greeting",
|
|
"objective": "Respond to hello"},), {}),
|
|
("_te_job_to_dict", ({"job_id": "m2", "task_family": "architecture",
|
|
"objective": WIDE, "inputs": {"path": "src/x.py"},
|
|
"required_output_schema": "diff", "risk": "high",
|
|
"reversibility": "irreversible",
|
|
"verification_method": "pytest",
|
|
"preferred_tiers": ["T0", "T1"], "max_tier": "T3"},), {}),
|
|
# ── CognitiveResult.to_dict via the real constructor ─────────────────────
|
|
("_te_result_to_dict", ({"job_id": "r1", "tier_used": "T0",
|
|
"model_used": "deterministic", "success": True},), {}),
|
|
("_te_result_to_dict", ({"job_id": "r2", "tier_used": "T1",
|
|
"model_used": "qwen2.5:3b", "success": False,
|
|
"output": {"response": "partial"},
|
|
"error": "boom", "verification_result": {"ok": 1},
|
|
"attempts": 3, "latency_ms": 12.5,
|
|
"tokens_used": 42},), {}),
|
|
# ── execute(): deterministic family paths (registry default, T0) ─────────
|
|
("_te_execute", (None, {"job_id": "e1", "task_family": "greeting",
|
|
"objective": "hi there"}, BUDGET_T0), {}),
|
|
("_te_execute", (None, {"job_id": "e2", "task_family": "lookup",
|
|
"objective": "find the config"}, BUDGET_T0), {}),
|
|
("_te_execute", (None, {"job_id": "e3", "task_family": "coding",
|
|
"objective": "run the build"}, BUDGET_T0), {}),
|
|
("_te_execute", (None, {"job_id": "e4", "task_family": "greeting",
|
|
"objective": LONG_ASCII_105}, BUDGET_T0), {}),
|
|
("_te_execute", (None, {"job_id": "e5", "task_family": "greeting",
|
|
"objective": "日本語" * 40}, BUDGET_T0), {}),
|
|
("_te_execute", (None, {"job_id": "e6", "task_family": "lookup",
|
|
"objective": "héllo wörld café", "required_output_schema": "y"},
|
|
BUDGET_T0), {}),
|
|
# preferred_tiers honored before the budget start tier
|
|
("_te_execute", (None, {"job_id": "e7", "task_family": "greeting",
|
|
"objective": "pref first",
|
|
"preferred_tiers": ["T0"]}, BUDGET_T2), {}),
|
|
# ── execute(): no-profile early return (subset registry, no profiles) ──
|
|
("_te_execute", (EMPTY_REG, {"job_id": "e8", "task_family": "coding",
|
|
"objective": "x", "preferred_tiers": ["T3"]},
|
|
BUDGET_T3), {}),
|
|
("_te_execute", (EMPTY_REG, {"job_id": "e9", "task_family": "coding",
|
|
"objective": "x"}, BUDGET_T3), {}),
|
|
# ── execute(): model paths against the rig's OLLAMA echo server ─────────
|
|
("_te_execute_ollama", (None, None, {"job_id": "m1", "task_family": "coding",
|
|
"objective": "fix it"}, BUDGET_T1), {}),
|
|
("_te_execute_ollama", (None, None, {"job_id": "m2", "task_family": "coding",
|
|
"objective": WIDE},
|
|
BUDGET_T1), {}),
|
|
("_te_execute_ollama", (None, None, {"job_id": "m3", "task_family": "coding",
|
|
"objective": "preferred tier",
|
|
"preferred_tiers": ["T2"]},
|
|
{"start_tier": "T0", "max_tier": "T4"},
|
|
), {}),
|
|
("_te_execute_ollama", (None, {"T3": T3_PROFILE}, {"job_id": "m4",
|
|
"task_family": "coding",
|
|
"objective": "subset reg"},
|
|
BUDGET_T3), {}),
|
|
("_te_execute_ollama", (None, None, {"job_id": "m5", "task_family": "coding",
|
|
"objective": "cloud tier"},
|
|
BUDGET_T4), {}),
|
|
# ── execute(): crash-parity rows through the model path ─────────────────
|
|
("_te_execute_ollama", ("definitely-not-a-url", None,
|
|
{"job_id": "x1", "task_family": "coding",
|
|
"objective": "bad scheme"}, BUDGET_T1), {}),
|
|
("_te_execute_ollama", ("http://127.0.0.1:9", None,
|
|
{"job_id": "x2", "task_family": "coding",
|
|
"objective": "refused"}, BUDGET_T1), {}),
|
|
# ── _ctx_budget (vendored resolve_context_budget surface) ───────────────
|
|
("_ctx_budget", ("qwen2.5:3b",), {}),
|
|
("_ctx_budget", ("qwen3:14b",), {}),
|
|
("_ctx_budget", ("qwen3.6:27b",), {}),
|
|
("_ctx_budget", ("mimo-v2.5-pro",), {}),
|
|
("_ctx_budget", ("",), {}),
|
|
("_ctx_budget", ("no-such-model",), {}),
|
|
# ── test_executor (assert-safe capture; echo server answers the T2 job) ──
|
|
("_te_test_executor", (), {}),
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
truth = freeze("rune.tiered_executor", CASES, oracle_module="rune.tiered_executor")
|
|
write_envelope(FROZEN_NOW, truth, OUT)
|
|
|
|
by_fn = {}
|
|
for row in truth:
|
|
by_fn.setdefault(row["fn"], []).append(row)
|
|
assert len(truth) == len(CASES), f"freeze produced {len(truth)} rows for {len(CASES)} cases"
|
|
assert by_fn["_te_job_to_dict"][0]["expected"]["inputs"] == {}
|
|
assert by_fn["_te_result_to_dict"][1]["expected"]["verification_result"] == {"ok": 1}
|
|
assert by_fn["_te_execute"][0]["expected"]["output"] == {"message": "Hello! Processing: hi there"}
|
|
assert len(by_fn["_te_execute"][3]["expected"]["output"]["message"]) == 119
|
|
assert len(by_fn["_te_execute"][4]["expected"]["output"]["message"]) == 119
|
|
assert by_fn["_te_execute"][7]["expected"]["error"] == "No profile for tier T3"
|
|
assert by_fn["_te_execute"][8]["expected"]["error"] == "No profile for tier T3"
|
|
echo_row = by_fn["_te_execute_ollama"][0]["expected"]
|
|
assert echo_row["success"] is True and echo_row["tier_used"] == "T1"
|
|
assert echo_row["output"]["response"].startswith('{"model": "qwen2.5:3b", "prompt": "fix it"')
|
|
assert by_fn["_te_execute_ollama"][2]["expected"]["tier_used"] == "T2"
|
|
assert by_fn["_te_execute_ollama"][3]["expected"]["model_used"] == "qwen3.6:27b"
|
|
assert by_fn["_te_execute_ollama"][4]["expected"]["model_used"] == "mimo-v2.5-pro"
|
|
trunc_row = by_fn["_te_execute_ollama"][1]["expected"]
|
|
assert len(trunc_row["output"]["response"]) == 2000, len(trunc_row["output"]["response"])
|
|
assert by_fn["_te_execute_ollama"][5]["expected"]["error"].startswith("unknown url type")
|
|
assert by_fn["_te_execute_ollama"][6]["expected"]["error"] == "<urlopen error [Errno 111] Connection refused>"
|
|
assert by_fn["_ctx_budget"][0]["expected"] == 30720
|
|
assert by_fn["_ctx_budget"][3]["expected"] == 124928
|
|
assert by_fn["_te_test_executor"][0]["expected"] == {"returned": None}
|
|
|
|
print(f"wrote {OUT}: {len(truth)} cases, frozen_now={FROZEN_NOW}")
|
|
for fn, rows in by_fn.items():
|
|
print(f" {fn}: {len(rows)} row(s)")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main()) |