197 lines
7.0 KiB
Python
197 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Live parity: subconscious_pass Rust kernel vs Python oracle (dichotomy
|
|
wave). Layers: FROZEN replay (22 cases through oracle floor AND kernel),
|
|
LIVE facade-vs-oracle, FUZZ (600 seeded random turns, seed 2479).
|
|
`sp.datetime` is frozen so report timestamps and save filenames are
|
|
byte-comparable.
|
|
"""
|
|
import copy
|
|
import datetime as _dt
|
|
import json
|
|
import os
|
|
import random
|
|
import string
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
os.environ["RUNE_LEDGER_CORE"] = "python"
|
|
|
|
import rune.subconscious_pass as sp # noqa: E402
|
|
from rune import sp_loader # noqa: E402
|
|
|
|
FIXED_DT = _dt.datetime(2026, 10, 12, 0, 0, 1, 1, tzinfo=_dt.timezone.utc)
|
|
|
|
|
|
class _FixedDatetime(_dt.datetime):
|
|
@classmethod
|
|
def now(cls, tz=None):
|
|
return FIXED_DT
|
|
|
|
|
|
sp.datetime = _FixedDatetime
|
|
|
|
TRUTH = json.loads(
|
|
(REPO / "src" / "rust" / "subconscious_pass" / "src"
|
|
/ "frozen_sp_truth.json").read_text())
|
|
|
|
fails = []
|
|
counts = {"frozen": 0, "live": 0, "fuzz": 0}
|
|
|
|
WORDS = ("build fix cache index deploy planning drift frustrated confused "
|
|
"loader test guard topic refactor shipping agent task".split())
|
|
|
|
|
|
def rand_text(rnd, n=None):
|
|
n = n if n is not None else rnd.randint(1, 40)
|
|
txt = " ".join(rnd.choice(WORDS) for _ in range(n))
|
|
if rnd.random() < 0.2:
|
|
txt += "?"
|
|
return txt
|
|
|
|
|
|
def fail(layer, name, detail):
|
|
fails.append((layer, name, detail))
|
|
print(f"FAIL[{layer}] {name}: {detail}")
|
|
|
|
|
|
def frozen():
|
|
for group, cases in TRUTH.items():
|
|
if group in ("module", "now"):
|
|
continue
|
|
for c in cases:
|
|
counts["frozen"] += 1
|
|
args = copy.deepcopy(c["args"])
|
|
fn = c["fn"]
|
|
if fn == "reflect":
|
|
s = sp.SubconsciousPass()
|
|
want = c["expect"]
|
|
got_o = s.reflect(args["user"], args["agent"],
|
|
args.get("context"))
|
|
got_k = sp_loader.reflect(args["user"], args["agent"],
|
|
args.get("context"), args["now"])
|
|
elif fn == "extract_topic":
|
|
s = sp.SubconsciousPass()
|
|
want = c["expect"]
|
|
got_o = s._extract_topic(args["text"])
|
|
got_k = sp_loader.extract_topic(args["text"])
|
|
elif fn == "recent_findings":
|
|
s = sp.SubconsciousPass()
|
|
s.findings = list(args["reports"])
|
|
want = c["expect"]
|
|
got_o = s.get_recent_findings(args.get("limit", 10))
|
|
got_k = sp_loader.recent_findings(args["reports"],
|
|
args.get("limit", 10))
|
|
elif fn == "save_reflection":
|
|
want = c["expect"]
|
|
got_o = json.dumps(args["report"], indent=2)
|
|
got_k = sp_loader.save_report(args["report"])
|
|
else: # pragma: no cover
|
|
continue
|
|
for tag, got in (("oracle", got_o), ("kernel", got_k)):
|
|
if json.dumps(got, sort_keys=True) != json.dumps(
|
|
want, sort_keys=True):
|
|
fail("frozen", f"{fn}/{c['name']}/{tag}",
|
|
f"want={json.dumps(want, sort_keys=True)[:260]} "
|
|
f"got={json.dumps(got, sort_keys=True)[:260]}")
|
|
|
|
|
|
def compare(name, o, k, layer="live"):
|
|
counts[layer] += 1
|
|
jo, jk = json.dumps(o, sort_keys=True), json.dumps(k, sort_keys=True)
|
|
if jo != jk:
|
|
fail(layer, name, f"oracle={jo[:260]} kernel={jk[:260]}")
|
|
|
|
|
|
def run_both(user, agent, ctx, findings):
|
|
so = sp.SubconsciousPass()
|
|
sk = sp.SubconsciousPass()
|
|
so.findings = list(findings)
|
|
sk.findings = list(findings)
|
|
os.environ["RUNE_SP_CORE"] = "python"
|
|
o = {"reflect": so.reflect(user, agent, ctx),
|
|
"extract": so._extract_topic(user),
|
|
"recent10": so.get_recent_findings(),
|
|
"recent0": so.get_recent_findings(0),
|
|
"recent3": so.get_recent_findings(3),
|
|
"recent_neg2": so.get_recent_findings(-2)}
|
|
os.environ["RUNE_SP_CORE"] = "auto"
|
|
k = {"reflect": sk.reflect(user, agent, ctx),
|
|
"extract": sk._extract_topic(user),
|
|
"recent10": sk.get_recent_findings(),
|
|
"recent0": sk.get_recent_findings(0),
|
|
"recent3": sk.get_recent_findings(3),
|
|
"recent_neg2": sk.get_recent_findings(-2)}
|
|
return o, k
|
|
|
|
|
|
def live_and_fuzz():
|
|
rnd = random.Random(2479)
|
|
findings = [{"findings": [{"type": "a", "detail": "d", "severity": "i"}]},
|
|
{"findings": []},
|
|
{"other": 1}]
|
|
# live on the frozen inputs
|
|
for group, cases in TRUTH.items():
|
|
if group in ("module", "now") or group == "save_reflection_cases":
|
|
continue
|
|
for c in cases:
|
|
args = c["args"]
|
|
o, k = run_both(args.get("user", args.get("text", "")),
|
|
args.get("agent", "ok"),
|
|
args.get("context"), findings)
|
|
for key in o:
|
|
compare(f"{c['name']}/{key}", o[key], k[key])
|
|
# fuzz
|
|
with tempfile.TemporaryDirectory(prefix="sp_parity_") as tmp:
|
|
for i in range(600):
|
|
user = rand_text(rnd, rnd.choice([1, 3, 30, 45]))
|
|
agent = rand_text(rnd, rnd.choice([1, 5, 12]))
|
|
ctx = None
|
|
if rnd.random() < 0.6:
|
|
ctx = {}
|
|
if rnd.random() < 0.7:
|
|
ctx["previous_topic"] = rnd.choice(
|
|
["", "index tuning", rand_text(rnd, 4)])
|
|
o, k = run_both(user, agent, ctx, findings)
|
|
for key in o:
|
|
compare(f"fuzz{i}/{key}", o[key], k[key], layer="fuzz")
|
|
# save_reflection byte parity through the real oracle path
|
|
report = o["reflect"]
|
|
so = sp.SubconsciousPass()
|
|
sk = sp.SubconsciousPass()
|
|
orig_dir = sp.REFLECTIONS_DIR
|
|
try:
|
|
for tag, inst in (("o", so), ("k", sk)):
|
|
d = Path(tmp) / f"{i}{tag}"
|
|
d.mkdir()
|
|
sp.REFLECTIONS_DIR = str(d)
|
|
inst.save_reflection(report)
|
|
counts["fuzz"] += 1
|
|
fo = sorted((Path(tmp) / f"{i}o").iterdir())[0]
|
|
fk = sorted((Path(tmp) / f"{i}k").iterdir())[0]
|
|
if fo.name != fk.name or fo.read_text() != fk.read_text():
|
|
fail("fuzz", f"fuzz{i}/save", "file bytes differ")
|
|
finally:
|
|
sp.REFLECTIONS_DIR = orig_dir
|
|
|
|
|
|
def main():
|
|
if not sp_loader.available():
|
|
print("kernel not built — build src/rust/subconscious_pass first")
|
|
return 1
|
|
os.environ["RUNE_SP_CORE"] = "python"
|
|
frozen()
|
|
live_and_fuzz()
|
|
os.environ["RUNE_SP_CORE"] = "auto"
|
|
total = counts["frozen"] + counts["live"] + counts["fuzz"]
|
|
print(f"parity {total - len(fails)}/{total} "
|
|
f"(frozen {counts['frozen']}, live {counts['live']}, "
|
|
f"fuzz {counts['fuzz']})")
|
|
return 1 if fails else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|