209 lines
8.1 KiB
Python
209 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""gen_outcome_ledger_truth.py — outcome_ledger dichotomy wave truth battery
|
|
(#2479 tranche). Drives the LIVE python oracle (RUNE_OL_CORE=python pinned)
|
|
through every public function and branch, and freezes the observations to
|
|
src/rust/outcome_ledger/src/frozen_to_truth.json.
|
|
|
|
Public surface exercised:
|
|
* OutcomeLedger.__init__ (fresh db + reopen-existing branches)
|
|
* OutcomeLedger.record (all field branches: success/fail, correction,
|
|
floats, defaults)
|
|
* OutcomeLedger.get_stats (no-family branch AND task_family branch,
|
|
empty-db zero branch AND populated branch)
|
|
* OutcomeLedger.get_model_stats (hit and miss)
|
|
* OutcomeRecord.to_dict (full record)
|
|
(test_ledger stays a facade-level python self-test — covered by live
|
|
parity, not by the kernel battery.)
|
|
|
|
Timestamps are PINNED in the inputs, so observations are fully
|
|
deterministic and need no masking.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
sys.path.insert(0, str(REPO / "src" / "rune"))
|
|
os.environ["RUNE_OL_CORE"] = "python"
|
|
|
|
from rune.outcome_ledger import OutcomeLedger, OutcomeRecord # noqa: E402
|
|
|
|
T0 = "2026-10-11T00:00:01+00:00"
|
|
T1 = "2026-10-11T00:00:02+00:00"
|
|
T2 = "2026-10-11T00:00:03+00:00"
|
|
T3 = "2026-10-11T00:00:04+00:00"
|
|
T4 = "2026-10-11T00:00:05+00:00"
|
|
T5 = "2026-10-11T00:00:06+00:00"
|
|
|
|
|
|
class Driver:
|
|
"""Drives op scripts against the live oracle the way the truth was
|
|
frozen — each scenario gets its own temp db path."""
|
|
|
|
def __init__(self, workdir: Path, n: int):
|
|
self.path = str(workdir / f"ol_{n}.db")
|
|
self.ledger = None
|
|
|
|
def step(self, op: dict):
|
|
kind = op["op"]
|
|
if kind == "new":
|
|
if not op.get("reuse"):
|
|
self.ledger = OutcomeLedger(self.path)
|
|
else:
|
|
# fresh instance over the SAME db (CREATE TABLE IF NOT
|
|
# EXISTS idempotent branch)
|
|
self.ledger = OutcomeLedger(self.path)
|
|
return {"result": None}
|
|
if kind == "record":
|
|
self.ledger.record(OutcomeRecord(**op["record"]))
|
|
return {"result": None}
|
|
if kind == "to_dict":
|
|
return {"result": OutcomeRecord(**op["record"]).to_dict()}
|
|
if kind == "get_stats":
|
|
fam = op.get("family")
|
|
if fam is None:
|
|
return {"result": self.ledger.get_stats()}
|
|
return {"result": self.ledger.get_stats(fam)}
|
|
if kind == "get_model_stats":
|
|
return {"result": self.ledger.get_model_stats(op["model"])}
|
|
raise ValueError(f"unknown op {kind}")
|
|
|
|
|
|
def s1_empty_db(workdir):
|
|
"""Empty-database branches of every stats path."""
|
|
d = Driver(workdir, 1)
|
|
ops = [
|
|
{"op": "new"},
|
|
{"op": "get_stats"},
|
|
{"op": "get_stats", "family": "greeting"},
|
|
{"op": "get_model_stats", "model": "qwen2.5:3b"},
|
|
]
|
|
return {"name": "s1_empty_db", "ops": ops, "obs": [d.step(o) for o in ops]}
|
|
|
|
|
|
def s2_record_and_stats(workdir):
|
|
"""The oracle's own test_ledger data, plus every stats entry point."""
|
|
d = Driver(workdir, 2)
|
|
ops = [
|
|
{"op": "new"},
|
|
{"op": "record", "record": {
|
|
"timestamp": T0, "task_family": "greeting", "model": "qwen2.5:3b",
|
|
"provider": "ollama", "tier": "T1", "task_success": True,
|
|
"latency_ms": 100.0}},
|
|
{"op": "record", "record": {
|
|
"timestamp": T1, "task_family": "coding", "model": "qwen3:14b",
|
|
"provider": "ollama", "tier": "T2", "task_success": True,
|
|
"attempts": 2, "latency_ms": 500.0}},
|
|
{"op": "record", "record": {
|
|
"timestamp": T2, "task_family": "coding", "model": "qwen3:14b",
|
|
"provider": "ollama", "tier": "T2", "task_success": False,
|
|
"user_correction": True, "latency_ms": 600.0}},
|
|
{"op": "get_stats", "family": "greeting"},
|
|
{"op": "get_stats", "family": "coding"},
|
|
{"op": "get_stats"},
|
|
{"op": "get_model_stats", "model": "qwen3:14b"},
|
|
{"op": "get_model_stats", "model": "unknown-model"},
|
|
]
|
|
return {"name": "s2_record_and_stats", "ops": ops, "obs": [d.step(o) for o in ops]}
|
|
|
|
|
|
def s3_full_record_to_dict(workdir):
|
|
"""OutcomeRecord.to_dict with every field set; float-heavy stats."""
|
|
d = Driver(workdir, 3)
|
|
full = {
|
|
"timestamp": T3, "task_family": "review", "model": "gpt-5-mini",
|
|
"provider": "openai", "tier": "T3", "prompt_tokens": 1200,
|
|
"output_tokens": 340, "cached_tokens": 512, "latency_ms": 123.75,
|
|
"attempts": 1, "verification_result": "pass", "user_correction": False,
|
|
"task_success": True, "estimated_cost": 0.0025,
|
|
}
|
|
ops = [
|
|
{"op": "new"},
|
|
{"op": "to_dict", "record": full},
|
|
{"op": "record", "record": full},
|
|
{"op": "record", "record": {**full, "timestamp": T4,
|
|
"estimated_cost": 0.0075,
|
|
"verification_result": "fail",
|
|
"task_success": False}},
|
|
{"op": "get_stats"},
|
|
{"op": "get_stats", "family": "review"},
|
|
{"op": "get_model_stats", "model": "gpt-5-mini"},
|
|
]
|
|
return {"name": "s3_full_record_to_dict", "ops": ops, "obs": [d.step(o) for o in ops]}
|
|
|
|
|
|
def s4_persistence_reopen(workdir):
|
|
"""A fresh OutcomeLedger over an existing db — rows survive, DDL is
|
|
idempotent."""
|
|
d = Driver(workdir, 4)
|
|
ops = [
|
|
{"op": "new"},
|
|
{"op": "record", "record": {
|
|
"timestamp": T0, "task_family": "port", "model": "qwen2.5:3b",
|
|
"provider": "ollama", "tier": "T1", "task_success": True,
|
|
"latency_ms": 42.5}},
|
|
{"op": "record", "record": {
|
|
"timestamp": T1, "task_family": "port", "model": "qwen3:14b",
|
|
"provider": "ollama", "tier": "T2", "task_success": False,
|
|
"user_correction": True, "attempts": 3, "latency_ms": 900.0}},
|
|
{"op": "new", "reuse": True},
|
|
{"op": "get_stats"},
|
|
{"op": "get_stats", "family": "port"},
|
|
{"op": "get_model_stats", "model": "qwen2.5:3b"},
|
|
]
|
|
return {"name": "s4_persistence_reopen", "ops": ops, "obs": [d.step(o) for o in ops]}
|
|
|
|
|
|
def s5_averages_and_rates(workdir):
|
|
"""AVG/rate arithmetic: fractional averages, 0%/50%/100% rates, both
|
|
correction branches."""
|
|
d = Driver(workdir, 5)
|
|
ops = [
|
|
{"op": "new"},
|
|
{"op": "record", "record": {
|
|
"timestamp": T0, "task_family": "ops", "model": "m1",
|
|
"task_success": True, "attempts": 1, "latency_ms": 10.5,
|
|
"estimated_cost": 0.0}},
|
|
{"op": "record", "record": {
|
|
"timestamp": T1, "task_family": "ops", "model": "m1",
|
|
"task_success": False, "user_correction": True, "attempts": 2,
|
|
"latency_ms": 20.25, "estimated_cost": 1.5}},
|
|
{"op": "record", "record": {
|
|
"timestamp": T2, "task_family": "ops", "model": "m2",
|
|
"task_success": True, "attempts": 3, "latency_ms": 30.125,
|
|
"user_correction": True, "estimated_cost": 2.25}},
|
|
{"op": "get_stats"},
|
|
{"op": "get_model_stats", "model": "m1"},
|
|
{"op": "get_model_stats", "model": "m2"},
|
|
{"op": "get_stats", "family": "ops"},
|
|
]
|
|
return {"name": "s5_averages_and_rates", "ops": ops, "obs": [d.step(o) for o in ops]}
|
|
|
|
|
|
def main():
|
|
out = {
|
|
"module": "outcome_ledger",
|
|
"seed": 2479,
|
|
"oracle": "src/rune/outcome_ledger.py",
|
|
"pin": "RUNE_OL_CORE=python",
|
|
"generated": "2026-10-11",
|
|
"scenarios": [],
|
|
}
|
|
with tempfile.TemporaryDirectory() as td:
|
|
workdir = Path(td)
|
|
for gen in (s1_empty_db, s2_record_and_stats, s3_full_record_to_dict,
|
|
s4_persistence_reopen, s5_averages_and_rates):
|
|
out["scenarios"].append(gen(workdir))
|
|
dest = REPO / "src" / "rust" / "outcome_ledger" / "src" / "frozen_to_truth.json"
|
|
dest.write_text(json.dumps(out, indent=1) + "\n")
|
|
n_obs = sum(len(s["obs"]) for s in out["scenarios"])
|
|
print(f"wrote {len(out['scenarios'])} scenarios / {n_obs} observations to {dest}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|