Files
rune/tests/test_pl1_records.py
T
Nick a125c023e6
CI Pipeline / best-practices (push) Successful in 8s
CI Pipeline / documentation (push) Successful in 4s
CI Pipeline / canary (push) Successful in 35s
CI Pipeline / lint (push) Successful in 44s
Auto-version and Release / release (push) Successful in 49s
CI Pipeline / type-check (push) Successful in 31s
CI Pipeline / gitea-offline (push) Successful in 17s
CI Pipeline / unit-tests (push) Successful in 1m1s
deploy / deploy (push) Failing after 15m38s
CI Pipeline / required-checks (push) Successful in 1s
feat(PL1): first tranche — structured records, failure classification, route-instruction learning loop, metrics
PL1-01: scripts/pl1_records.py — compact structured Experience evidence from
canonical task state + route.jsonl (route resolution by task_id+timestamp,
proof level, failure class, candidate status).
PL1-02: failure classes in the PL0 vocabulary, evidence-grounded; UNKNOWN
valid; research_dispatch never failure.
PL1-03: first learning loop — recurring prose-instead-of-mutation family →
structural correction (route-driven instruction appended to the user context
block for mutation routes) + replay contract (NOT_COMPLETED gate) —
recorded as PE-1789102658-0269 (SEED; held-out = next deployed mutation tasks).
PL1-05: metrics module (first_pass_accepted, false_completion_rate=0.0
structurally, mutation_failures, research_dispatches, candidate/evidence split).

Intent state updated: PL1 first tranche live; ADAPT-LIVE acceptance proven on
deployed runtime; remaining ADAPT obligations deferred to their epics.
2026-09-11 05:58:47 +01:00

127 lines
4.7 KiB
Python

"""PL1-01/02/03 tests: structured records, failure classes, route instruction."""
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path
PKG_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(PKG_ROOT / "src"))
sys.path.insert(0, str(PKG_ROOT / "scripts"))
from pl1_records import ( # noqa: E402
build_eligible_records,
enrich_record,
failure_class,
resolve_route,
)
class ResolveRoute(unittest.TestCase):
def test_resolves_by_task_and_timestamp(self):
with tempfile.TemporaryDirectory() as tmp:
log = Path(tmp) / "route.jsonl"
log.write_text(
json.dumps(
{
"kind": "INITIAL_ROUTE",
"route": "LOCAL_FIX",
"decided_at": "2026-09-11T01:00:00+00:00",
"dispatch": {"task_id": "t1"},
}
)
+ "\n"
)
got = resolve_route("t1", "route:2026-09-11T01:00:00+00:00", str(log))
self.assertEqual(got, "LOCAL_FIX")
def test_unresolvable_is_empty_never_guessed(self):
self.assertEqual(resolve_route("tX", "route:nope", "/nonexistent"), "")
self.assertEqual(resolve_route("tX", "", "/nonexistent"), "")
class FailureClassification(unittest.TestCase):
def test_research_dispatch_not_failure(self):
t = {"status": "research_required", "outcome_ref": "research_dispatch"}
self.assertEqual(failure_class(t), "research_dispatch")
def test_not_completed_contract(self):
t = {"status": "failed", "result": "NOT_COMPLETED_NO_MUTATION_EVIDENCE: ..."}
self.assertEqual(failure_class(t), "NOT_COMPLETED_NO_MUTATION_EVIDENCE")
def test_verification_failure(self):
t = {"status": "failed", "result": "verification failed: exit 1"}
self.assertEqual(failure_class(t), "verification_failure")
def test_completed_has_no_failure_class(self):
self.assertEqual(failure_class({"status": "completed", "result": "done"}), "UNKNOWN")
class EligibleRecords(unittest.TestCase):
def test_builds_from_checkpoint(self):
with tempfile.TemporaryDirectory() as tmp:
state = {
"tasks": [
{
"task_id": "t1",
"title": "fix bug",
"status": "failed",
"result": "NOT_COMPLETED_NO_MUTATION_EVIDENCE: x",
"route_ref": "",
"assigned_model": "m",
"outcome_ref": "",
},
{
"task_id": "t2",
"title": "chat",
"status": "completed",
"result": "hello",
"route_ref": "",
"assigned_model": "m",
},
]
}
sp = Path(tmp) / "task_state.json"
sp.write_text(json.dumps(state))
recs = build_eligible_records(str(sp))
self.assertEqual(len(recs), 2)
by_id = {r["task_id"]: r for r in recs}
self.assertEqual(by_id["t1"]["failure_class"], "NOT_COMPLETED_NO_MUTATION_EVIDENCE")
self.assertEqual(by_id["t1"]["candidate_status"], "CANDIDATE")
self.assertEqual(by_id["t2"]["candidate_status"], "EVIDENCE_ONLY")
class RouteInstruction(unittest.TestCase):
def test_mutation_routes_carry_instruction(self):
import importlib.util
spec = importlib.util.spec_from_file_location("rune_operator", PKG_ROOT / "src" / "rune" / "rune_operator.py")
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
from rune import adaptive_router
op = m.RuneOperator.__new__(m.RuneOperator)
for text in (
"Fix the failing assertion in tests/test_x.py",
"Add retry logic to the client and update tests/test_client.py",
):
s = adaptive_router.derive_sufficiency(text[:200], text)
d = adaptive_router.classify_route(
text,
has_local_repro=(s.local_authority == "GROUNDED"),
owning_authority_known=True,
testable_locally=(s.verifiability == "TESTABLE"),
narrow_and_reversible=(s.scope_novelty == "KNOWN_PATTERN"),
sufficiency=s,
)
ctx = {"route": d.route}
block = op._route_instruction(d, ctx)
self.assertTrue(block, f"no instruction for {d.route}")
self.assertIn("MUST use", block)
if __name__ == "__main__":
unittest.main()