Nick
bd5c46f9d9
Kernel/facade split per the intent ledger: the pure kernels (analyze probe→findings threshold ladder, suggest ⚠️/ℹ️/❌ mapping, hook passthrough record, pinned-clock _now isoformat kernel, CLI rendering switch) live in src/rust/self_improve behind the doc-envelope FFI (si_analyze/si_suggest/si_handle_hook/si_now/si_main); all five file/DB seams stay in the Python facade probe builder with error strings crossing FFI verbatim. si_loader ctypes bridge (RUNE_SI_CORE=python pin, SI_RUST_LIB override, shared cs_free_string allocator). Frozen truth: 61 rows (26 analyze / 10 suggest / 4 hook / 8 now / 13 cli) generated from the live oracle with path-free fixture messages, pinned clock, and exact-representable :.0% ratios; 61/61 byte-identical at replay across facade both-pins + kernel direct; branch_battery replays the frozen file natively; pytest port battery 8/8; clippy -D warnings clean.
744 lines
28 KiB
Python
744 lines
28 KiB
Python
#!/usr/bin/env python3
|
||
"""gen_self_improve_truth.py — freeze/replay parity truth for self_improve
|
||
(dichotomy wave, #2479 tranche).
|
||
|
||
Freeze mode (default): exercises EVERY public function of the LIVE Python
|
||
oracle (src/rune/self_improve.py) across every branch and records
|
||
(kernel_input, expected_output) pairs into
|
||
src/rust/self_improve/src/frozen_si_truth.json:
|
||
|
||
* analyze — kernel probe -> findings (+ the stderr warning lines the
|
||
oracle body prints for failed seams)
|
||
* suggest — synthetic findings -> suggestion strings (⚠️/ℹ️/❌ rows,
|
||
KeyError raise-parity rows, unknown-severity no-op row); the oracle's
|
||
suggest_improvements() takes no arguments (it calls analyze_system),
|
||
so the seam injection pins analyze_system to the synthetic findings —
|
||
the suggest LOGIC itself is always the live oracle's
|
||
* handle_hook — passthrough record rows
|
||
* now — pinned-clock `_now()` rows (epoch microseconds -> the
|
||
datetime.now(timezone.utc).isoformat() byte shape)
|
||
* cli — the `main` switch end to end (analyze/analyze --json/
|
||
suggest/hook over real fixtures) with the EXACT stdout/stderr byte
|
||
streams
|
||
|
||
Replay mode (--replay): re-runs the SAME scenarios and asserts both
|
||
kernel-level (loader direct) and facade-level parity against the frozen
|
||
file, in pin mode (RUNE_SI_CORE=python — the original body) AND delegation
|
||
mode (auto — the Rust kernel via si_loader).
|
||
|
||
Poisoned-truth law: freeze pins RUNE_SI_CORE=python BEFORE any import.
|
||
|
||
No-path-truth law: fixtures that fail do so with path-free sqlite
|
||
messages ("unable to open database file", "no such table: entries",
|
||
"Expecting value: line 1 column 1 (char 0)"). The missing-.microkernel.json
|
||
case is deliberately NOT frozen — its FileNotFoundError message embeds the
|
||
machine path (truth law violation); the same code branch is covered by the
|
||
invalid-JSON case.
|
||
|
||
No-wall-clock law: `_now` rows freeze the clock via a pinned datetime
|
||
subclass (freeze_truth.py's _frozen_clock pattern); the kernel takes the
|
||
instant as epoch microseconds — an argument, never the wall clock.
|
||
|
||
Deterministic-fixture law: fixtures are built from declarative specs
|
||
(fixed row counts, sqlite executemany) so freeze, replay and the Rust
|
||
battery agree on every count. The Rust battery needs NO fixtures —
|
||
analyze/cli rows carry the probe itself as the kernel input.
|
||
|
||
:.0% law: cache rows use exact-representable ratios (halves/quarters/
|
||
eighths/thirty-seconds) so python's :.0% (exact-decimal scaling,
|
||
round-half-to-even) and the Rust {:.0}% (binary f64 multiply, round-half-
|
||
even) agree by construction — 1/8 → 12.5% → "12%", 3/8 → 37.5% → "38%".
|
||
"""
|
||
import copy
|
||
import io
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
from contextlib import contextmanager, redirect_stderr, redirect_stdout
|
||
from datetime import datetime, timezone
|
||
|
||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
TRUTH_PATH = os.path.join(
|
||
REPO, "src", "rust", "self_improve", "src", "frozen_si_truth.json"
|
||
)
|
||
REPLAY = "--replay" in sys.argv
|
||
|
||
os.environ["RUNE_SI_CORE"] = "python" # poisoned-truth law (freeze mode)
|
||
|
||
sys.path.insert(0, os.path.join(REPO, "src"))
|
||
sys.path.insert(0, os.path.join(REPO, "src", "rune"))
|
||
|
||
import self_improve as oracle_mod # noqa: E402
|
||
|
||
if REPLAY:
|
||
import si_loader # noqa: E402 (kernel-level replay side)
|
||
|
||
EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||
_ORACLE_ANALYZE = oracle_mod.analyze_system # seam-injection anchor
|
||
|
||
# ── fixture builder (declarative, deterministic) ─────────────────────────────
|
||
|
||
|
||
def _fill(cur, table, cols, rows):
|
||
marks = ",".join("?" * len(cols))
|
||
cur.executemany(
|
||
f"INSERT INTO {table} ({','.join(cols)}) VALUES ({marks})", rows
|
||
)
|
||
|
||
|
||
def _store_db(spec, root):
|
||
"""The memory-store sqlite: entries(status)/chunks/associations and
|
||
use_history. Missing tables / missing parent dir per spec."""
|
||
store = spec.get("store", {})
|
||
no_dir = store.get("no_dir")
|
||
db_path = (os.path.join(root, "no_store", "store.db") if no_dir
|
||
else os.path.join(root, "store.db"))
|
||
if no_dir:
|
||
return db_path # parent intentionally absent → connect fails
|
||
conn = sqlite3.connect(db_path)
|
||
cur = conn.cursor()
|
||
missing = set(store.get("missing", []))
|
||
if "entries" not in missing:
|
||
cur.execute("CREATE TABLE entries (id INTEGER PRIMARY KEY, status TEXT)")
|
||
_fill(cur, "entries", ["id", "status"],
|
||
[(i, "active") for i in range(store.get("active", 0))]
|
||
+ [(10_000 + i, "archived")
|
||
for i in range(store.get("archived", 0))])
|
||
if "chunks" not in missing:
|
||
cur.execute("CREATE TABLE chunks (id INTEGER PRIMARY KEY, entry_id INTEGER)")
|
||
_fill(cur, "chunks", ["id"], [(i,) for i in range(store.get("chunks", 0))])
|
||
if "associations" not in missing:
|
||
cur.execute("CREATE TABLE associations (id INTEGER PRIMARY KEY, a INTEGER)")
|
||
_fill(cur, "associations", ["id"],
|
||
[(i,) for i in range(store.get("associations", 0))])
|
||
if "use_history" not in missing:
|
||
cur.execute("CREATE TABLE use_history (id INTEGER PRIMARY KEY)")
|
||
_fill(cur, "use_history", ["id"], [(i,) for i in range(store.get("history", 0))])
|
||
conn.commit()
|
||
conn.close()
|
||
return db_path
|
||
|
||
|
||
def _plugins_file(spec, root):
|
||
plugins = spec.get("plugins", {})
|
||
scripts = os.path.join(root, "scripts")
|
||
os.makedirs(scripts, exist_ok=True)
|
||
if plugins.get("absent"):
|
||
return # branch covered via invalid-json only (path-free law)
|
||
count = plugins.get("count", 0)
|
||
reg = {
|
||
"plugins": {f"p{i:03d}": {"name": f"p{i:03d}"} for i in range(count)},
|
||
"hooks": {"on_message": [], "on_startup": []},
|
||
}
|
||
payload = json.dumps(reg) if not plugins.get("invalid") else "{not-json"
|
||
with open(os.path.join(scripts, ".microkernel.json"), "w") as f:
|
||
f.write(payload)
|
||
|
||
|
||
def _abstraction_db(spec, root):
|
||
ab = spec.get("abstraction", {})
|
||
if ab.get("no_file"):
|
||
return # connect creates an empty db → "no such table: facts"
|
||
path = os.path.join(root, "memory", "abstraction.db")
|
||
conn = sqlite3.connect(path)
|
||
cur = conn.cursor()
|
||
cur.execute("CREATE TABLE facts (id INTEGER PRIMARY KEY, state TEXT)")
|
||
_fill(cur, "facts", ["id", "state"],
|
||
[(i, "active") for i in range(ab.get("facts", 0))]
|
||
+ [(9_000 + i, "archived")
|
||
for i in range(ab.get("facts_inactive", 0))])
|
||
cur.execute("CREATE TABLE observations (id INTEGER PRIMARY KEY, state TEXT)")
|
||
_fill(cur, "observations", ["id", "state"],
|
||
[(i, "active") for i in range(ab.get("obs", 0))])
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def _cache_db(spec, root):
|
||
cache = spec.get("cache", {})
|
||
if cache.get("no_file"):
|
||
return # connect creates an empty db → "no such table: cache"
|
||
path = os.path.join(root, "memory", "cache.db")
|
||
conn = sqlite3.connect(path)
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"CREATE TABLE cache (key TEXT PRIMARY KEY, invalidated_at TEXT,"
|
||
" hit_count INTEGER, miss_count INTEGER)")
|
||
rows = cache.get("rows", [])
|
||
_fill(cur, "cache", ["key", "invalidated_at", "hit_count", "miss_count"],
|
||
[(f"k{i:04d}", None if inv == 0 else "2026-01-01T00:00:00Z", h, m)
|
||
for i, (h, m, inv) in enumerate(rows)])
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def build_fixture(spec, root):
|
||
"""Spec → deterministic fixture tree under root. Returns the STORE_DB
|
||
path the facade must be pinned to."""
|
||
os.makedirs(os.path.join(root, "memory"), exist_ok=True)
|
||
_plugins_file(spec, root)
|
||
_abstraction_db(spec, root)
|
||
_cache_db(spec, root)
|
||
return _store_db(spec, root)
|
||
|
||
|
||
@contextmanager
|
||
def pin(store_path, root):
|
||
"""Pin the oracle's three seams to the fixture: STORE_DB / SCRIPTS_DIR
|
||
module attrs + RUNE_MEMORY_DIR env (memory_env reads it at call
|
||
time). Restores everything after."""
|
||
old = (oracle_mod.STORE_DB, oracle_mod.SCRIPTS_DIR,
|
||
os.environ.get("RUNE_MEMORY_DIR"))
|
||
oracle_mod.STORE_DB = store_path
|
||
oracle_mod.SCRIPTS_DIR = os.path.join(root, "scripts")
|
||
os.environ["RUNE_MEMORY_DIR"] = os.path.join(root, "memory")
|
||
try:
|
||
yield
|
||
finally:
|
||
oracle_mod.STORE_DB, oracle_mod.SCRIPTS_DIR = old[0], old[1]
|
||
if old[2] is None:
|
||
os.environ.pop("RUNE_MEMORY_DIR", None)
|
||
else:
|
||
os.environ["RUNE_MEMORY_DIR"] = old[2]
|
||
|
||
|
||
def run_with_pin(spec, root, fn):
|
||
store = build_fixture(spec, root)
|
||
with pin(store, root):
|
||
return fn()
|
||
|
||
|
||
def frozen_datetime(iso):
|
||
base = datetime.fromisoformat(iso)
|
||
|
||
class FrozenDatetime(datetime):
|
||
@classmethod
|
||
def now(cls, tz=None):
|
||
if tz is not None:
|
||
return base.replace(tzinfo=tz)
|
||
return base
|
||
|
||
return FrozenDatetime
|
||
|
||
|
||
def micros_of(iso):
|
||
dt = datetime.fromisoformat(iso).replace(tzinfo=timezone.utc)
|
||
delta = dt - EPOCH
|
||
return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
|
||
|
||
|
||
def _catch(fn, *a):
|
||
try:
|
||
return fn(*a)
|
||
except KeyError as e:
|
||
return {"__raised__": f"KeyError: {e}"}
|
||
except json.JSONDecodeError as e:
|
||
return {"__raised__": f"JSONDecodeError: {e}"}
|
||
except ValueError as e:
|
||
return {"__raised__": f"ValueError: {e}"}
|
||
|
||
|
||
# ── scenario capture ─────────────────────────────────────────────────────────
|
||
|
||
|
||
def capture_analyze(name, spec):
|
||
def _go():
|
||
probe = copy.deepcopy(oracle_mod._probe())
|
||
err = io.StringIO()
|
||
with redirect_stderr(err):
|
||
findings = copy.deepcopy(oracle_mod.analyze_system())
|
||
return {"name": name, "fixture": spec, "probe": probe,
|
||
"findings": findings, "stderr": err.getvalue()}
|
||
return _go
|
||
|
||
|
||
def capture_cli(name, spec, tail):
|
||
def _go():
|
||
probe = copy.deepcopy(oracle_mod._probe())
|
||
out, err = io.StringIO(), io.StringIO()
|
||
old_argv = sys.argv
|
||
sys.argv = ["self_improve.py"] + tail
|
||
try:
|
||
with redirect_stdout(out), redirect_stderr(err):
|
||
raised = _catch(oracle_mod.main)
|
||
finally:
|
||
sys.argv = old_argv
|
||
row = {"name": name, "argv": tail, "fixture": spec, "probe": probe,
|
||
"stdout": out.getvalue(), "stderr": err.getvalue()}
|
||
if raised is not None:
|
||
row["raises"] = raised["__raised__"]
|
||
return row
|
||
return _go
|
||
|
||
|
||
def suggest_row(name, findings):
|
||
def _go():
|
||
oracle_mod.analyze_system = lambda: copy.deepcopy(findings)
|
||
try:
|
||
expected = _catch(oracle_mod.suggest_improvements)
|
||
finally:
|
||
oracle_mod.analyze_system = _ORACLE_ANALYZE
|
||
return {"name": name, "findings": findings, "expected": expected}
|
||
return _go
|
||
|
||
|
||
def hook_row(name, event, data):
|
||
def _go():
|
||
return {"name": name, "event": event, "data": data,
|
||
"expected": oracle_mod.handle_hook(event, copy.deepcopy(data))}
|
||
return _go
|
||
|
||
|
||
def now_row(name, iso):
|
||
def _go():
|
||
oracle_mod.datetime = frozen_datetime(iso)
|
||
try:
|
||
expected = oracle_mod._now()
|
||
finally:
|
||
oracle_mod.datetime = datetime
|
||
return {"name": name, "iso": iso, "micros": micros_of(iso),
|
||
"expected": expected}
|
||
return _go
|
||
|
||
|
||
# ── fixture specs ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def spec(**kw):
|
||
base = {
|
||
"store": {"active": 100, "archived": 2, "chunks": 200,
|
||
"associations": 60, "history": 150},
|
||
"plugins": {"count": 10},
|
||
"abstraction": {"facts": 51, "facts_inactive": 3, "obs": 10},
|
||
"cache": {"rows": [[7, 3, 0] for _ in range(10)]},
|
||
}
|
||
for k, v in kw.items():
|
||
base[k].update(v)
|
||
return base
|
||
|
||
|
||
def cache_rows(hits, misses, n=1):
|
||
return [[hits, misses, 0] for _ in range(n)]
|
||
|
||
|
||
ANALYZE_CASES = [
|
||
("healthy", spec()),
|
||
("entries-1000-edge", spec(store={"active": 1000})),
|
||
("entries-1001-warning", spec(store={"active": 1001})),
|
||
("sparse-edge-5of10", spec(store={"active": 10, "chunks": 20,
|
||
"associations": 5})),
|
||
("sparse-4of10", spec(store={"active": 10, "chunks": 20,
|
||
"associations": 4})),
|
||
("big-and-sparse", spec(store={"active": 1100, "chunks": 900,
|
||
"associations": 540})),
|
||
("plugins-80-edge", spec(plugins={"count": 80})),
|
||
("plugins-81", spec(plugins={"count": 81})),
|
||
("plugins-invalid-json", spec(plugins={"invalid": True})),
|
||
("facts-0", spec(abstraction={"facts": 0, "facts_inactive": 0, "obs": 5})),
|
||
("facts-5-obs-0", spec(abstraction={"facts": 5, "facts_inactive": 0,
|
||
"obs": 0})),
|
||
("facts-50-edge", spec(abstraction={"facts": 50, "facts_inactive": 0,
|
||
"obs": 0})),
|
||
("scaled-everything", spec(
|
||
store={"active": 1100, "chunks": 900, "associations": 540,
|
||
"history": 99},
|
||
plugins={"count": 85},
|
||
abstraction={"facts": 40, "facts_inactive": 2, "obs": 0},
|
||
cache={"rows": cache_rows(1, 8, 9)})),
|
||
("cache-quarter", spec(cache={"rows": cache_rows(1, 3)})),
|
||
("cache-tie-12-and-half", spec(cache={"rows": cache_rows(1, 7)})),
|
||
("cache-tie-37-and-half", spec(cache={"rows": [[1, 1, 0]] + [[1, 1, 0],
|
||
[1, 1, 0], [0, 1, 0],
|
||
[0, 1, 0]]})),
|
||
("cache-low-3pct", spec(cache={"rows": cache_rows(1, 31)})),
|
||
("cache-half-no-warn", spec(cache={"rows": cache_rows(1, 1)})),
|
||
("cache-zero-hits", spec(cache={"rows": cache_rows(0, 10)})),
|
||
("cache-null-sums", spec(cache={"rows": [[None, None, 0],
|
||
[None, None, 1]]})),
|
||
("cache-no-table", spec(cache={"no_file": True})),
|
||
("history-99", spec(store={"history": 99})),
|
||
("history-100-edge", spec(store={"history": 100})),
|
||
("store-error-missing-table",
|
||
spec(store={"active": 0, "chunks": 0, "associations": 0, "history": 5,
|
||
"missing": ["entries", "chunks", "associations"]})),
|
||
("store-error-no-dir", spec(store={"no_dir": True, "history": 0})),
|
||
("warn-no-tables", spec(abstraction={"no_file": True},
|
||
cache={"no_file": True})),
|
||
]
|
||
|
||
SPEC_BY_NAME = {name: fs for name, fs in ANALYZE_CASES}
|
||
|
||
SUGGEST_CASES = [
|
||
("mixed-severities", [
|
||
{"area": "memory_store", "severity": "warning",
|
||
"finding": "Memory store has 1100 entries — approaching scale "
|
||
"where retrieval quality may degrade",
|
||
"suggestion": "Consider implementing abstraction pipeline to "
|
||
"compress entries into concepts"},
|
||
{"area": "plugins", "severity": "info",
|
||
"finding": "85 plugins registered — capability broker should "
|
||
"help discovery",
|
||
"suggestion": "Verify capability broker is matching intents to "
|
||
"plugins correctly"},
|
||
{"area": "memory_store", "severity": "error",
|
||
"finding": "unable to open database file"},
|
||
]),
|
||
("empty-findings", []),
|
||
("error-only", [{"area": "memory_store", "severity": "error",
|
||
"finding": "no such table: entries"}]),
|
||
("unknown-severity-skip", [
|
||
{"area": "cache", "severity": "critical", "finding": "f",
|
||
"suggestion": "s"},
|
||
{"area": "cache", "severity": "info", "finding": "f2",
|
||
"suggestion": "s2"},
|
||
]),
|
||
("bool-severity-truthy", [
|
||
{"area": "cache", "severity": True, "finding": "f",
|
||
"suggestion": "s"},
|
||
]),
|
||
("missing-suggestion", [
|
||
{"area": "cache", "severity": "warning",
|
||
"finding": "Cache hit rate is 0% — low effectiveness"},
|
||
]),
|
||
("missing-severity", [{"area": "cache", "finding": "f"}]),
|
||
("zero-severity", [
|
||
{"area": "cache", "severity": 0, "finding": "f", "suggestion": "s"},
|
||
]),
|
||
("missing-area-error-branch", [
|
||
{"severity": "error", "finding": "f"},
|
||
]),
|
||
("missing-finding-warning-branch", [
|
||
{"area": "cache", "severity": "warning", "suggestion": "s"},
|
||
]),
|
||
]
|
||
|
||
HOOK_CASES = [
|
||
("default-data", "cycle", None),
|
||
("with-data", "cycle", {"k": 1}),
|
||
("empty-event", "", {"a": []}),
|
||
("data-ignored-string", "e", "not-a-dict"),
|
||
]
|
||
|
||
NOW_CASES = [
|
||
("epoch-zero", "1970-01-01T00:00:00+00:00"),
|
||
("2026-no-micros", "2026-10-11T12:00:00+00:00"),
|
||
("2026-micros", "2026-10-11T12:00:00.000123+00:00"),
|
||
("leap-day-2020", "2020-02-29T23:59:59.999999+00:00"),
|
||
("year-boundary", "2026-12-31T23:59:59.999999+00:00"),
|
||
("year-1", "0001-06-15T12:00:00+00:00"),
|
||
("year-9999", "9999-12-31T23:59:59.000001+00:00"),
|
||
("pre-epoch", "1969-12-31T23:59:59.999999+00:00"),
|
||
]
|
||
|
||
CLI_CASES = [
|
||
("cli-analyze-healthy-empty", "healthy", ["analyze"]),
|
||
("cli-analyze-scaled", "scaled-everything", ["analyze"]),
|
||
("cli-analyze-scaled-json", "scaled-everything", ["analyze", "--json"]),
|
||
("cli-analyze-store-error", "store-error-no-dir", ["analyze"]),
|
||
("cli-analyze-warn-no-tables", "warn-no-tables", ["analyze"]),
|
||
("cli-analyze-json-healthy", "healthy", ["analyze", "--json"]),
|
||
("cli-suggest-healthy", "healthy", ["suggest"]),
|
||
("cli-suggest-scaled", "scaled-everything", ["suggest"]),
|
||
("cli-suggest-store-error", "store-error-no-dir", ["suggest"]),
|
||
("cli-hook-default", "healthy", ["hook"]),
|
||
("cli-hook-event", "healthy", ["hook", "--event", "cycle"]),
|
||
("cli-hook-data", "healthy", ["hook", "--event", "e", "--data", '{"a": 1}']),
|
||
("cli-hook-bad-data", "healthy", ["hook", "--data", "{bad"]),
|
||
]
|
||
|
||
# ── freeze / replay driver ───────────────────────────────────────────────────
|
||
|
||
|
||
def jbytes(v):
|
||
return json.dumps(v, sort_keys=True, ensure_ascii=True)
|
||
|
||
|
||
_RECS: dict = {}
|
||
|
||
|
||
def _record(group, name, row, frozen):
|
||
_RECS.setdefault(group, []).append(row)
|
||
if not REPLAY:
|
||
return True
|
||
old = next((c for c in frozen.get(group, []) if c["name"] == name), None)
|
||
if old is None:
|
||
print(f"NEW ROW [{group}] {name} — not in frozen truth (re-freeze first)")
|
||
return False
|
||
diverged = False
|
||
for key in row:
|
||
if key in old and jbytes(row[key]) != jbytes(old[key]):
|
||
print(f"FACADE DIVERGE [{group}] {name} {key}")
|
||
print(" got: ", jbytes(row[key])[:600])
|
||
print(" want:", jbytes(old[key])[:600])
|
||
diverged = True
|
||
for key in old:
|
||
if key not in row:
|
||
print(f"FACADE DIVERGE [{group}] {name} missing key {key}")
|
||
diverged = True
|
||
return not diverged
|
||
|
||
|
||
def _check(group, name, got, want, label="KERNEL"):
|
||
if isinstance(want, dict) and want.get("__raised__"):
|
||
ok = isinstance(got, dict) and got.get("__raised__") == want["__raised__"]
|
||
elif isinstance(want, dict) and ("stdout" in want or "stderr" in want):
|
||
ok = isinstance(got, dict) and got.get("stdout") == want.get("stdout") \
|
||
and got.get("stderr") == want.get("stderr")
|
||
else:
|
||
ok = jbytes(got) == jbytes(want)
|
||
if not ok:
|
||
print(f"{label} DIVERGE [{group}] {name}")
|
||
print(" got: ", json.dumps(got, sort_keys=True, default=str)[:600])
|
||
print(" want:", json.dumps(want, sort_keys=True, default=str)[:600])
|
||
return ok
|
||
|
||
|
||
def _py_pin():
|
||
os.environ["RUNE_SI_CORE"] = "python"
|
||
|
||
|
||
def _rust_pin():
|
||
if not si_loader.available():
|
||
_py_pin()
|
||
return False
|
||
os.environ.pop("RUNE_SI_CORE", None)
|
||
return True
|
||
|
||
|
||
def _restore_pin(prev):
|
||
if prev is None:
|
||
os.environ.pop("RUNE_SI_CORE", None)
|
||
else:
|
||
os.environ["RUNE_SI_CORE"] = prev
|
||
|
||
|
||
def _replay_analyze(row, frozen):
|
||
group, name = "analyze", row["name"]
|
||
old = next(c for c in frozen[group] if c["name"] == name)
|
||
ok = True
|
||
for pin_label in ("python-pin", "rust-pin"):
|
||
prev = os.environ.get("RUNE_SI_CORE")
|
||
if pin_label == "python-pin":
|
||
_py_pin()
|
||
elif not _rust_pin():
|
||
print(f"SKIP [{name}] rust-pin (kernel not built)")
|
||
continue
|
||
|
||
def _run():
|
||
probe = copy.deepcopy(oracle_mod._probe())
|
||
err = io.StringIO()
|
||
with redirect_stderr(err):
|
||
findings = copy.deepcopy(oracle_mod.analyze_system())
|
||
return probe, findings, err.getvalue()
|
||
|
||
try:
|
||
probe2, findings2, stderr2 = run_with_pin(
|
||
row["fixture"], tempfile.mkdtemp(prefix=f"si_rp_{name}_"), _run)
|
||
finally:
|
||
_restore_pin(prev)
|
||
got = {"probe": probe2, "findings": findings2, "stderr": stderr2}
|
||
for key in ("probe", "findings", "stderr"):
|
||
if jbytes(got[key]) != jbytes(old[key]):
|
||
print(f"FACADE DIVERGE [{pin_label}] {name} {key}")
|
||
print(" got: ", jbytes(got[key])[:600])
|
||
print(" want:", jbytes(old[key])[:600])
|
||
ok = False
|
||
k_out = si_loader.analyze(copy.deepcopy(old["probe"]))
|
||
if jbytes(k_out) != jbytes(old["findings"]):
|
||
print(f"KERNEL DIVERGE [analyze] {name}")
|
||
print(" got: ", jbytes(k_out)[:600])
|
||
print(" want:", jbytes(old["findings"])[:600])
|
||
ok = False
|
||
return ok
|
||
|
||
|
||
def _replay_simple(group, row, old, py_fn, kernel_fn):
|
||
"""Two tiers: python-pin facade call, rust-pin facade call. Both go
|
||
through the same seam the production path uses (kernel_fn IS the
|
||
loader call the facade delegates to)."""
|
||
prev = os.environ.get("RUNE_SI_CORE")
|
||
_py_pin()
|
||
try:
|
||
py_out = _catch(py_fn, row)
|
||
finally:
|
||
_restore_pin(prev)
|
||
ok1 = _check(group, row["name"], py_out, old["expected"], "FACADE-py")
|
||
prev = os.environ.get("RUNE_SI_CORE")
|
||
_py_pin()
|
||
restored = True
|
||
try:
|
||
restored = _rust_pin()
|
||
k_out = _catch(kernel_fn, row)
|
||
finally:
|
||
_restore_pin(prev)
|
||
if not restored:
|
||
print(f"SKIP [{group}/{row['name']}] rust-pin (kernel not built)")
|
||
return ok1
|
||
ok2 = _check(group, row["name"], k_out, old["expected"], "FACADE-rust")
|
||
return ok1 and ok2
|
||
|
||
|
||
def _replay_now(row, old):
|
||
"""python-pin: pin the clock to the row's frozen instant (the live
|
||
oracle reads datetime.now(timezone.utc), never the wall truth)."""
|
||
def py_fn(r):
|
||
oracle_mod.datetime = frozen_datetime(r["iso"])
|
||
try:
|
||
return oracle_mod._now()
|
||
finally:
|
||
oracle_mod.datetime = datetime
|
||
prev = os.environ.get("RUNE_SI_CORE")
|
||
_py_pin()
|
||
try:
|
||
py_out = py_fn(row)
|
||
finally:
|
||
_restore_pin(prev)
|
||
ok1 = _check("now", row["name"], py_out, old["expected"], "FACADE-py")
|
||
k_out = si_loader.now(row["micros"])
|
||
ok2 = _check("now", row["name"], k_out, old["expected"], "KERNEL")
|
||
return ok1 and ok2
|
||
|
||
|
||
def _replay_cli(row, frozen):
|
||
group, name = "cli", row["name"]
|
||
old = next(c for c in frozen[group] if c["name"] == name)
|
||
ok = True
|
||
for pin_label in ("python-pin", "rust-pin"):
|
||
prev = os.environ.get("RUNE_SI_CORE")
|
||
if pin_label == "python-pin":
|
||
_py_pin()
|
||
elif not _rust_pin():
|
||
print(f"SKIP [cli-{name}] rust-pin (kernel not built)")
|
||
continue
|
||
try:
|
||
got = run_with_pin(
|
||
row["fixture"], tempfile.mkdtemp(prefix=f"si_rc_{name}_"),
|
||
capture_cli(row["name"], row["fixture"], row["argv"]))
|
||
finally:
|
||
_restore_pin(prev)
|
||
for key in ("probe", "stdout", "stderr", "raises"):
|
||
if key in old and jbytes(got.get(key)) != jbytes(old.get(key)):
|
||
print(f"FACADE DIVERGE [{pin_label}] cli {name} {key}")
|
||
print(" got: ", repr(got.get(key))[:300])
|
||
print(" want:", repr(old.get(key))[:300])
|
||
ok = False
|
||
if old.get("raises"):
|
||
return ok
|
||
# kernel direct: si_main over the frozen probe == frozen streams
|
||
argv = row["argv"]
|
||
event = argv[argv.index("--event") + 1] if "--event" in argv else None
|
||
k_out = si_loader.cli_main(argv[0], event or "unknown", None,
|
||
"--json" in argv, probe=row["probe"])
|
||
want = {"stdout": old["stdout"], "stderr": old["stderr"]}
|
||
return _check("cli", name, k_out, want) and ok
|
||
|
||
|
||
def main():
|
||
frozen = None
|
||
if REPLAY:
|
||
with open(TRUTH_PATH) as f:
|
||
frozen = json.load(f)["cases"]
|
||
|
||
failures = 0
|
||
total = 0
|
||
|
||
for name, fspec in ANALYZE_CASES:
|
||
total += 1
|
||
row = run_with_pin(fspec, tempfile.mkdtemp(prefix=f"si_{name}_"),
|
||
capture_analyze(name, fspec))
|
||
ok = _record("analyze", name, row, frozen)
|
||
if REPLAY:
|
||
ok = _replay_analyze(row, frozen) and ok
|
||
failures += 0 if ok else 1
|
||
|
||
for name, findings in SUGGEST_CASES:
|
||
total += 1
|
||
row = suggest_row(name, findings)()
|
||
ok = _record("suggest", name, row, frozen)
|
||
if REPLAY:
|
||
old = next(c for c in frozen["suggest"] if c["name"] == name)
|
||
|
||
def py_fn(r):
|
||
# the oracle's suggest_improvements() computes its own
|
||
# findings — inject the synthetic row (same seam patch the
|
||
# freeze used) so the oracle logic itself is what's replayed
|
||
oracle_mod.analyze_system = lambda: copy.deepcopy(r["findings"])
|
||
try:
|
||
return oracle_mod.suggest_improvements()
|
||
finally:
|
||
oracle_mod.analyze_system = _ORACLE_ANALYZE
|
||
|
||
ok = _replay_simple("suggest", row, old, py_fn,
|
||
lambda r: si_loader.suggest(r["findings"])) and ok
|
||
failures += 0 if ok else 1
|
||
|
||
for name, event, data in HOOK_CASES:
|
||
total += 1
|
||
row = hook_row(name, event, data)()
|
||
ok = _record("handle_hook", name, row, frozen)
|
||
if REPLAY:
|
||
old = next(c for c in frozen["handle_hook"] if c["name"] == name)
|
||
ok = _replay_simple(
|
||
"handle_hook", row, old,
|
||
py_fn=lambda r: oracle_mod.handle_hook(
|
||
r["event"], copy.deepcopy(r["data"])),
|
||
kernel_fn=lambda r: si_loader.handle_hook(r["event"], r["data"]),
|
||
) and ok
|
||
failures += 0 if ok else 1
|
||
|
||
for name, iso in NOW_CASES:
|
||
total += 1
|
||
row = now_row(name, iso)()
|
||
# micros cross the FFI as a string: years beyond ~285 CE exceed
|
||
# the f64 exact-integer range and would silently drop the
|
||
# sub-second digits (proven: year-9999 lost .000001 through
|
||
# f64). The truth row keeps BOTH forms; the string is law.
|
||
row["micros_str"] = str(row["micros"])
|
||
ok = _record("now", name, row, frozen)
|
||
if REPLAY:
|
||
old = next(c for c in frozen["now"] if c["name"] == name)
|
||
ok = _replay_now(row, old) and ok
|
||
failures += 0 if ok else 1
|
||
|
||
for name, fspec_name, tail in CLI_CASES:
|
||
total += 1
|
||
fspec = SPEC_BY_NAME[fspec_name]
|
||
row = run_with_pin(fspec, tempfile.mkdtemp(prefix=f"si_{name}_"),
|
||
capture_cli(name, fspec, tail))
|
||
ok = _record("cli", name, row, frozen)
|
||
if REPLAY:
|
||
ok = _replay_cli(row, frozen) and ok
|
||
failures += 0 if ok else 1
|
||
|
||
if REPLAY:
|
||
print(f"replay: {total - failures}/{total} rows byte-identical "
|
||
f"(facade both-pins + kernel direct)")
|
||
return 1 if failures else 0
|
||
|
||
doc = {
|
||
"frozen": datetime.now(timezone.utc).isoformat(),
|
||
"oracle": "src/rune/self_improve.py",
|
||
"pin": "RUNE_SI_CORE=python",
|
||
"note": "Frozen via scripts/gen_self_improve_truth.py against the "
|
||
"live Python oracle (RUNE_SI_CORE=python). Fixtures are "
|
||
"declarative specs with fixed row counts; every failing "
|
||
"fixture fails with PATH-FREE messages (the missing-"
|
||
".microkernel.json case is excluded — its FileNotFoundError "
|
||
"embeds the machine path). _now rows freeze the clock via "
|
||
"a pinned datetime (wall clock never trusted). Battery "
|
||
"embeds this VERBATIM.",
|
||
"cases": _RECS,
|
||
}
|
||
os.makedirs(os.path.dirname(TRUTH_PATH), exist_ok=True)
|
||
with open(TRUTH_PATH, "w") as f:
|
||
json.dump(doc, f, indent=1, sort_keys=True)
|
||
f.write("\n")
|
||
print(f"frozen {total - failures} cases → "
|
||
f"{os.path.relpath(TRUTH_PATH, REPO)}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main()) |