206 lines
7.4 KiB
Python
206 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""gen_recall_uplift_truth.py — freeze the recall_uplift parity oracle
|
|
(dichotomy wave #2479 tranche).
|
|
|
|
Generator law: RUNE_RU_CORE=python pinned, inputs deep-copied, every
|
|
expand() branch exercised: empty query, expansion-map matching (k in q /
|
|
q in k / k in t), store pass (LIKE match, LIMIT 32, distinct-doc
|
|
co-occurrence, df<5% distinctive filter, 24-term cap), and the best-effort
|
|
degrade (no store / sqlite error → map terms only).
|
|
|
|
PORTABLE LAW: CPython set iteration is hash-random, and the oracle's
|
|
candidate sort is stable over dict insertion order — so the frozen corpus
|
|
gives every co-occurring candidate word a DISTINCT doc count (ties would
|
|
make even Python-vs-Python output order nondeterministic).
|
|
|
|
Output: src/rust/recall_uplift/frozen_ru_truth.json — replayed VERBATIM by
|
|
the crate's cfg(test) battery.
|
|
"""
|
|
import copy
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
|
|
os.environ["RUNE_RU_CORE"] = "python"
|
|
|
|
from rune import recall_uplift # noqa: E402
|
|
|
|
cases: list = []
|
|
|
|
|
|
def freeze(query: str, corpus, expected):
|
|
cases.append({
|
|
"fn": "expand",
|
|
"args": {"query": query, "corpus": copy.deepcopy(corpus)},
|
|
"expected": copy.deepcopy(expected),
|
|
})
|
|
|
|
|
|
def build_db(rows):
|
|
"""rows: list[str|None] — active entry texts in insertion order."""
|
|
fd, path = tempfile.mkstemp(suffix=".db")
|
|
os.close(fd)
|
|
os.unlink(path)
|
|
db = sqlite3.connect(path)
|
|
db.execute("CREATE TABLE entries (id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
|
"text TEXT, status TEXT)")
|
|
for text in rows:
|
|
db.execute("INSERT INTO entries (text, status) VALUES (?, 'active')",
|
|
(text,))
|
|
db.commit()
|
|
db.close()
|
|
return path
|
|
|
|
|
|
def fetch_corpus(path):
|
|
db = sqlite3.connect(path)
|
|
try:
|
|
return [r[0] for r in db.execute(
|
|
"SELECT text FROM entries WHERE status='active'").fetchall()]
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def case_store(path, query):
|
|
"""Freeze one store-backed case: RUNE_STORE_DB pointed at the fixture,
|
|
corpus embedded verbatim in args."""
|
|
old = os.environ.get("RUNE_STORE_DB")
|
|
os.environ["RUNE_STORE_DB"] = path
|
|
try:
|
|
corpus = fetch_corpus(path)
|
|
expected = recall_uplift.expand(query)
|
|
finally:
|
|
if old is None:
|
|
os.environ.pop("RUNE_STORE_DB", None)
|
|
else:
|
|
os.environ["RUNE_STORE_DB"] = old
|
|
freeze(query, corpus, expected)
|
|
|
|
|
|
def case_degraded(query):
|
|
"""Freeze the no-store degrade (RUNE_STORE_DB points nowhere)."""
|
|
old = os.environ.get("RUNE_STORE_DB")
|
|
os.environ["RUNE_STORE_DB"] = "/nonexistent/rune-truth/memory.db"
|
|
try:
|
|
expected = recall_uplift.expand(query)
|
|
finally:
|
|
if old is None:
|
|
os.environ.pop("RUNE_STORE_DB", None)
|
|
else:
|
|
os.environ["RUNE_STORE_DB"] = old
|
|
freeze(query, None, expected)
|
|
|
|
|
|
def main() -> None:
|
|
# ── map-only branches (no store) ─────────────────────────────────────
|
|
case_degraded("") # empty query → []
|
|
case_degraded(" ") # whitespace query → []
|
|
case_degraded("housing crisis") # k in q branch
|
|
case_degraded("hous") # q in k branch
|
|
case_degraded("routers and models") # k in t branch (router⊂routers)
|
|
case_degraded("zebra crossing") # no match at all → [zebra, crossing]
|
|
case_degraded("CI pipeline") # mixed-case key match, short tokens
|
|
# map-only over an existing-but-empty store (corpus=[] branch)
|
|
empty_db = build_db([])
|
|
case_store(empty_db, "housing crisis")
|
|
os.unlink(empty_db)
|
|
|
|
# ── store pass, engineered corpus A (200 rows, distinct co-occur counts)
|
|
# matched rows 0..11 (testing 0..3 / memory 4..8 / recall 8..11 — recall
|
|
# is a seen map term, so it co-occurs but is skipped). NO filler text in
|
|
# matched rows (filler would become a tie-counted co-occur candidate).
|
|
rows = ["filler word"] * 200
|
|
CO = [("allocator", 9), ("snapshot", 8), ("checkpoint", 7), ("ledger", 6),
|
|
("walrus", 3), ("quixotic", 2), ("zephyr", 1)]
|
|
for w, n in CO:
|
|
for r in range(n):
|
|
rows[r] = (rows[r] + f" {w}").strip()
|
|
for r in range(4):
|
|
rows[r] += " testing" # co-occur count 4
|
|
for r in range(4, 9):
|
|
rows[r] += " memory" # co-occur count 5
|
|
for r in range(8, 12):
|
|
rows[r] += " recall" # in seen → skipped
|
|
for r in range(4):
|
|
rows[r] += " pipeline" # w == t quirk: last token skipped
|
|
for r in range(5):
|
|
rows[r] += " other" # stopword → skipped
|
|
for r in range(6):
|
|
rows[r] += " retrieval" # already in seen → skipped
|
|
for r in range(20): # df 20/200 = 0.1 → filtered branch
|
|
rows[r] += " orbital" # co-occur 12 (distinct, top candidate)
|
|
rows[150] = None # NULL text: counts in total, never matches
|
|
db_a = build_db(rows)
|
|
case_store(db_a, "testing memory recall pipeline")
|
|
os.unlink(db_a)
|
|
|
|
# ── 24-term cap boundary (400 rows, co-occur counts 19..1, distinct) ──
|
|
rows = ["filler word"] * 400
|
|
for i in range(19): # word in (19-i) matched rows; letters
|
|
w = "kv" + chr(97 + i) * 2 # only — the tokenizer is [a-z']{4,}
|
|
for r in range(19 - i):
|
|
rows[r] = (rows[r] + f" {w}").strip()
|
|
for r in range(20):
|
|
rows[r] = f"housing policy{rows[r]}" # no filler in matched rows;
|
|
# policy co-occurs in all 20 matched rows → df 20/400 = 0.05 → filtered
|
|
db_b = build_db(rows)
|
|
case_store(db_b, "housing policy anchor")
|
|
os.unlink(db_b)
|
|
|
|
# ── LIMIT-32 branch (100 rows: matches beyond the first 32 don't co-occur)
|
|
rows = ["filler word"] * 100
|
|
for r in range(40):
|
|
rows[r] = "alpha"
|
|
for r in range(8):
|
|
rows[r] += " beta"
|
|
for r in range(3):
|
|
rows[r] += " delta"
|
|
for r in range(4):
|
|
rows[r] += " gamma" # last token → w == t skip
|
|
for r in range(33, 40):
|
|
rows[r] += " omega" # beyond the LIMIT-32 cut → not in co-occur
|
|
db_c = build_db(rows)
|
|
case_store(db_c, "alpha beta gamma")
|
|
os.unlink(db_c)
|
|
|
|
# ── sqlite-error degrade: db present, entries table missing ──────────
|
|
fd, bad = tempfile.mkstemp(suffix=".db")
|
|
os.close(fd)
|
|
os.unlink(bad)
|
|
db = sqlite3.connect(bad)
|
|
db.execute("CREATE TABLE other (x TEXT)")
|
|
db.commit()
|
|
db.close()
|
|
old = os.environ.get("RUNE_STORE_DB")
|
|
os.environ["RUNE_STORE_DB"] = bad
|
|
try:
|
|
expected = recall_uplift.expand("housing crisis")
|
|
finally:
|
|
if old is None:
|
|
os.environ.pop("RUNE_STORE_DB", None)
|
|
else:
|
|
os.environ["RUNE_STORE_DB"] = old
|
|
freeze("housing crisis", None, expected)
|
|
os.unlink(bad)
|
|
|
|
doc = {
|
|
"module": "recall_uplift",
|
|
"pin": "RUNE_RU_CORE=python",
|
|
"frozen_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"cases": cases,
|
|
}
|
|
out = ROOT / "src" / "rust" / "recall_uplift" / "frozen_ru_truth.json"
|
|
out.write_text(json.dumps(doc, indent=1) + "\n")
|
|
print(f"frozen {len(cases)} cases -> {out}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|