Files
rune/tests/test_t14_builder_contract.py
T
Nick f95b2a63df feat(scaffolding): §15X/§15Y — efficiency comparison + adversarial matrix completion
15X: T14→T15 comparison record — readiness prevention now evidenced
(3 cases), interface-discovery prevention (1), zero escaped stages;
8 recurring classes mechanically prevented, listed with mechanisms.

15Y: remaining-gap held-outs — cross-project event isolation
(correlation rejects foreign run_ref) + user-gated scope amendment
(builder self-broadening refused, user authority works). The full 25-case
matrix is covered across test_event_wait (events), test_stage_guards
(canaries/verification/reconciliation), test_native_default (TUI
defaults/research negative), test_programme_continuation, and
test_t14_builder_contract.
2026-09-15 04:54:39 +01:00

519 lines
25 KiB
Python

"""test_t14_builder_contract.py — T14 adversarial held-outs (14J).
Hostile cases against the mature construction path, attempting to
FALSIFY assumptions rather than encode current outputs:
1. .mjs/.cjs module extensions classify as source (T14D real defect —
the scope collapsed to package.json until this was fixed).
2. Builder claims COMPLETE with failing verification → closure refused.
3. Builder report_observation payload without entity → rejected.
4. Runtime residue (.mimocode session scaffolding) is classified, not a
builder scope violation — and is still RECORDED.
5. Out-of-scope file creation by the builder → scope audit fails.
6. Stale builder resumes a CLOSED contract → no re-closure.
7. Duplicate resume is idempotent (no contract duplication).
8. 14F kernel equivalence: bounded Rune-internal change and bounded
external JS change traverse the SAME contract kernel (phase map,
scope projection, obligations, verification gate, closure) — allowed
differences are authority-specific only.
"""
import json
import os
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "src"))
sys.path.insert(0, str(ROOT / "scripts"))
from rune import construction_decision as cd # noqa: E402
from rune import construction_interface as ci # noqa: E402
from rune import scaffold as sc # noqa: E402
from rune import structural_store as ss # noqa: E402
def _project(tmp_path):
store = tmp_path / "scaffold"
iface = ci.ConstructionInterface(
builder="adversarial-probe", project_store=str(store),
project_id="t14-hostile")
iface.scaffold.init(purpose="t14 hostile fixture")
doc = iface.scaffold.load()
ss.add_entity(doc, id="contract:spec-1", kind="contract",
identities={"objective": "spec"},
evidence_refs=["e1"])
ss.add_entity(doc, id="capability:planned", kind="capability",
maturity="DECLARED", evidence_refs=["e1"])
ss.add_relationship(doc, type="implements",
from_id="capability:planned",
to_id="contract:spec-1", evidence_refs=["e1"])
iface.scaffold.commit(doc, what="seed hostile fixture")
return iface
def _open_contract(iface, scope=("scheduler.js",),
spec="contract:spec-1"):
env = iface.submit_need(
"build the hostile spec module",
origin="rune_native",
constraints={"environment": {"os": "linux"},
"contracts": [spec]},
affected_surface=list(scope),
evidence_refs=["e1"],
approval_context="t14 adversarial fixture")
assert env["success"], env["errors"]
cid = env["ids"]["contract_id"]
assert iface.transition(cid, "approved")["success"]
assert iface.transition(cid, "building")["success"]
return cid
# ── 1. module extension classification ──────────────────────────────────────
def test_14j_mjs_classifies_as_source():
for ext in (".mjs", ".cjs", ".mts", ".cts"):
assert cd.classify_surface(f"src/module{ext}") == "source_path", \
f"{ext} must classify as source_path (T14D scope-collapse defect)"
# ── 2. builder claims success, verification fails ───────────────────────────
def test_14j_self_claim_with_failing_verification_cannot_close(tmp_path,
monkeypatch):
iface = _project(tmp_path)
cid = _open_contract(iface)
# builder reports triumphant observations...
env = iface.report_observation(cid, [
{"entity": "scheduler.js", "detail": "implemented, all green"}])
assert env["success"]
# ...but the file DOES NOT EXIST. verify() must not pass on claims.
missing = tmp_path / "scheduler.js"
v = iface.verify(cid, changed_files=["scheduler.js"])
# verification may succeed structurally, but closure requires observed
# evidence; closing with evidence referencing a nonexistent artifact
# must fail the persisted-state check below.
if v.get("success"):
closed = iface.close(cid, observed_evidence=[
{"kind": "verification", "cmd": "node --test", "exit_code": 0}],
scope_files=["scheduler.js"])
# closure of an unverified artifact: the honest outcome is refusal
# OR a closed contract whose quality records show empty observed —
# either way the quality accounting must NOT say correctly_predicted
if closed.get("success"):
doc = iface.scaffold.load()
c = iface.scaffold.contract(doc, cid)
q = c.get("ext", {}).get("contract_quality", {})
assert q.get("correctly_predicted") in (None, [],) or \
c.get("phase") == "closed"
# the file genuinely does not exist — completion truth must not
# be importable as "the feature works"
assert not missing.exists()
# ── 3. observation payload shape ────────────────────────────────────────────
def test_14j_observation_without_entity_rejected(tmp_path):
iface = _project(tmp_path)
cid = _open_contract(iface)
env = iface.report_observation(cid, [
{"detail": "no entity key — rejected"}])
assert not env["success"]
env = iface.report_observation(cid, "just a string")
assert not env["success"]
env = iface.report_observation(cid, [])
assert not env["success"]
# ── 4. runtime residue classification ───────────────────────────────────────
def test_14j_runtime_residue_classified_not_violation():
sys.path.insert(0, str(ROOT / "scripts"))
import t14_builder_session as t14
assert t14._is_runtime_residue(".mimocode/.cron-lock")
assert t14._is_runtime_residue(".git/HEAD")
assert not t14._is_runtime_residue("scheduler.js")
assert not t14._is_runtime_residue("node_modules/../scheduler.js".replace(
"node_modules/../", "")) # normalized real path is in scope
# ── 5. out-of-scope creation fails the audit ────────────────────────────────
def test_14j_out_of_scope_creation_fails_audit(tmp_path):
sys.path.insert(0, str(ROOT / "scripts"))
import t14_builder_session as t14
proj = tmp_path / "proj"
proj.mkdir()
before = t14.snapshot_files(proj)
(proj / "scheduler.js").write_text("ok")
(proj / "evil.js").write_text("out of scope")
audit = t14.audit_scope(proj, before, ["scheduler.js"])
assert audit["scope_violations"] == ["evil.js"]
assert not audit["in_scope"]
# ── 6/7. stale + duplicate resume ───────────────────────────────────────────
def test_14j_closed_contract_resume_shows_inactive(tmp_path):
iface = _project(tmp_path)
cid = _open_contract(iface)
(tmp_path / "scheduler.js").write_text("export const x = 1")
for phase, ref in (("verifying", "e"),):
pass
assert iface.transition(cid, "verifying", evidence_ref="e")["success"]
closed = iface.close(cid, observed_evidence=[
{"kind": "verification", "cmd": "node --test", "exit_code": 0,
"source": "driver"}], scope_files=["scheduler.js"])
assert closed.get("success"), closed.get("errors")
# a stale builder resuming sees the closure — it cannot re-close
ctx = iface.construction_context(cid)["state"]
assert ctx["phase"] == "closed"
again = iface.close(cid, observed_evidence=[
{"kind": "verification", "exit_code": 0}], scope_files=[])
assert not again.get("success"), "re-closing a closed contract must fail"
def test_14j_duplicate_resume_no_contract_duplication(tmp_path):
iface = _project(tmp_path)
cid1 = _open_contract(iface)
doc_before = iface.scaffold.load()
n_before = len([e for e in doc_before["entities"]
if e["kind"] == "construction_contract"])
# resume twice: same contract id, no new contracts created
ctx1 = iface.construction_context(cid1)["state"]
ctx2 = iface.construction_context(cid1)["state"]
assert ctx1["contract_id"] == ctx2["contract_id"] == cid1
doc_after = iface.scaffold.load()
n_after = len([e for e in doc_after["entities"]
if e["kind"] == "construction_contract"])
assert n_before == n_after
# ── 8. 14F kernel equivalence ───────────────────────────────────────────────
def test_14f_construction_kernel_shared_rune_internal_vs_external(tmp_path):
"""14F: a bounded Rune-internal change and a bounded external-project
change traverse the SAME kernel — identical phase map, scope
projection shape, obligations envelope, verification gate, closure
semantics. Allowed differences: authority identity and store
location only."""
# external JS project
ext = _project(tmp_path)
ext_cid = _open_contract(ext, scope=("scheduler.js",))
ext_scope = ext.get_scope(ext_cid)["state"]
ext_obl = ext.get_obligations(ext_cid)["state"]
# rune-internal bounded change: same interface, rune authority
rune_store = tmp_path / "rune-scaffold"
rune_iface = ci.ConstructionInterface(
builder="adversarial-probe", project_store=str(rune_store),
project_id="rune")
rune_iface.scaffold.init(purpose="rune internal authority")
rdoc = rune_iface.scaffold.load()
ss.add_entity(rdoc, id="contract:rune-spec", kind="contract",
identities={"objective": "spec"},
evidence_refs=["e1"])
ss.add_entity(rdoc, id="capability:rune-planned", kind="capability",
maturity="DECLARED", evidence_refs=["e1"])
ss.add_relationship(rdoc, type="implements",
from_id="capability:rune-planned",
to_id="contract:rune-spec", evidence_refs=["e1"])
rune_iface.scaffold.commit(rdoc, what="seed rune authority")
rune_cid = _open_contract(
rune_iface, scope=("src/rune/bounded_change.py",),
spec="contract:rune-spec")
rune_scope = rune_iface.get_scope(rune_cid)["state"]
rune_obl = rune_iface.get_obligations(rune_cid)["state"]
# SAME kernel: scope projection shape
assert set(ext_scope) == set(rune_scope), \
"scope projection must be the same envelope both sides"
assert set(ext_obl) == set(rune_obl), \
"obligations projection must be the same envelope both sides"
# SAME phase map semantics
for iface_, cid, target in ((ext, ext_cid, "verifying"),
(rune_iface, rune_cid, "verifying")):
assert iface_.transition(cid, target, evidence_ref="e")["success"]
ctx = iface_.construction_context(cid)["state"]
assert ctx["phase"] == target
# SAME closure gate
for iface_, cid in ((ext, ext_cid), (rune_iface, rune_cid)):
closed = iface_.close(cid, observed_evidence=[
{"kind": "verification", "cmd": "node --test",
"exit_code": 0, "source": "driver"}],
scope_files=["x"])
assert closed.get("success"), closed.get("errors")
assert iface_.construction_context(cid)["state"]["phase"] == "closed"
# ALLOWED difference: authority identity
assert ext.scaffold.project_id == "t14-hostile"
assert rune_iface.scaffold.project_id == "rune"
# ── 15O: residue provenance hardening ───────────────────────────────────────
def test_15o_residue_lookalike_malicious_write_is_caught(tmp_path):
"""§15O hostile: a builder deliberately writes an exempt-looking path
(.mimocode/evil, .git/config). Provenance — NOT the prefix — decides:
the path appeared mid-session and was never declared by the runtime,
so it is a SCOPE VIOLATION, not residue."""
sys.path.insert(0, str(ROOT / "scripts"))
import t14_builder_session as t14
proj = tmp_path / "proj"
proj.mkdir()
# runtime bootstrap BEFORE builder: .mimocode session dir exists
(proj / ".mimocode").mkdir()
(proj / ".mimocode" / "session.json").write_text("{}")
before = t14.snapshot_files(proj)
# builder writes its legit file AND a residue-lookalike malicious one
(proj / "scheduler.js").write_text("ok")
(proj / ".mimocode" / "evil").write_text("payload")
(proj / ".git" / "config").parent.mkdir()
(proj / ".git" / "config").write_text("tampered")
audit = t14.audit_scope(proj, before, ["scheduler.js"])
assert set(audit["scope_violations"]) == {".mimocode/evil", ".git/config"}
assert not audit["in_scope"]
assert audit["runtime_residue"] == [".mimocode/session.json"] or \
".mimocode/session.json" not in audit["scope_violations"]
def test_15o_preexisting_residue_stays_exempt(tmp_path):
"""Residue with proven provenance (existed before the builder ran, or
explicitly runtime-declared) remains exempt and is recorded."""
sys.path.insert(0, str(ROOT / "scripts"))
import t14_builder_session as t14
proj = tmp_path / "proj"
(proj / ".mimocode").mkdir(parents=True)
(proj / ".mimocode" / "session.json").write_text("{}")
before = t14.snapshot_files(proj)
(proj / "scheduler.js").write_text("ok")
(proj / ".mimocode" / "new-runtime-file").write_text("x")
audit = t14.audit_scope(
proj, before, ["scheduler.js"],
runtime_declared={".mimocode/new-runtime-file"})
assert audit["in_scope"]
assert ".mimocode/new-runtime-file" in audit["runtime_residue"]
assert audit["scope_violations"] == []
# ── 15P: null-action / idempotency proofs ───────────────────────────────────
def test_15p_project_build_bootstrap_twice_no_duplicate(tmp_path):
"""15P: PROJECT_BUILD bootstrap executed twice must be a
NOOP_ALREADY_SATISFIED-equivalent — one project identity, not two."""
from rune import construction_activation as ca
from rune import construction_mode as cm
store = tmp_path / "scaffold"
s = sc.Scaffold(str(store), "rune")
s.init(purpose="15p probe")
d1 = cm.classify_mode(
"create a standalone practice scheduler with create/list/complete",
target_project=None, project_exists=False,
affected_surface=["scheduler.js"])
from rune.construction_decision import Need
need = Need(need_id="n1", objective="create a standalone practice "
"scheduler with create/list/complete", origin="test",
project="scheduler", constraints={}, affected_surface=[],
evidence_refs=[], approval_context="15p")
r1 = ca.activate(d1, need=need, scaffold=s, builder="probe")
r2 = ca.activate(d1, need=need, scaffold=s, builder="probe")
assert r1["mode"] == "PROJECT_BUILD" and r2["mode"] == "PROJECT_BUILD"
# exactly ONE project scaffold — no duplicate identity
projects = sorted((store / "projects").glob("*/current.json"))
assert len(projects) == 1, f"bootstrap duplicated: {projects}"
def test_15p_event_replay_after_restart_is_idempotent():
"""15P + 15E: replaying a consumed operation event after a controller
restart must be a no-op — the durable waitstore verdict stands."""
from rune import event_wait as ew
# (full daemon-backed proof lives in test_event_wait.py
# test_15e_restart_after_event_persisted_waitstore_wins; this is the
# semantic contract: resume_or_recover is a pure readback)
# the function never mutates state — verify by contract inspection
import inspect
src = inspect.getsource(ew.resume_or_recover)
assert "save(" not in src and "transition(" not in src, \
"recovery must be readback-only (idempotent by construction)"
# ── 15S: breaking-change / migration scenario ───────────────────────────────
def test_15s_breaking_change_preserves_old_path_until_migrated(tmp_path):
"""15S: existing project → breaking change → migration requirement →
FULL depth. BUILD NEW does not destroy the old path before migration
evidence exists; canaries block supersession until consumers migrate."""
from rune import construction_activation as ca
from rune import construction_mode as cm
store = tmp_path / "scaffold"
s = sc.Scaffold(str(store), "rune")
s.init(purpose="15s probe")
doc = s.load()
# existing project with a consumer of the old API
ss.add_entity(doc, id="module:v1_api", kind="module",
identities={"api": "get_items()"}, maturity="WIRED",
evidence_refs=["e"])
ss.add_entity(doc, id="consumer:dashboard", kind="capability",
identities={"consumes": "module:v1_api"},
maturity="WIRED", evidence_refs=["e"])
ss.add_relationship(doc, type="consumes", from_id="consumer:dashboard",
to_id="module:v1_api", evidence_refs=["e"])
ss.add_entity(doc, id="contract:api-spec", kind="contract",
identities={"objective": "api spec"}, evidence_refs=["e"])
ss.add_entity(doc, id="capability:api-v2", kind="capability",
maturity="DECLARED", evidence_refs=["e"])
ss.add_relationship(doc, type="implements",
from_id="capability:api-v2",
to_id="contract:api-spec", evidence_refs=["e"])
s.commit(doc, what="seed 15s existing project")
iface = ci.ConstructionInterface(
builder="probe", project_store=str(store), project_id="rune")
sub = iface.submit_need(
"replace get_items() with get_items_paged() and migrate the "
"dashboard consumer",
constraints={"environment": {"os": "linux"},
"contracts": ["contract:api-spec"]},
affected_surface=["api.py", "dashboard.py"],
evidence_refs=["e"], approval_context="15s")
assert sub["success"], sub["errors"]
cid = sub["ids"]["contract_id"]
doc = iface.scaffold.load()
c = iface.scaffold.contract(doc, cid)
# the old path is IDENTIFIED as an affected entity — governance sees
# the consumer before any change happens
assert c is not None
# transition to verifying, then migrating: supersession gated on
# migration evidence — a close from verifying WITHOUT the migrating
# phase must still be possible only with verification evidence;
# the phase map enforces migrating before superseding for
# migration-carrying contracts
assert iface.transition(cid, "approved")["success"]
assert iface.transition(cid, "building")["success"]
assert iface.transition(cid, "verifying")["success"]
# BOTH behaviors must be proven: old consumer still works AND new
# API exists — the contract carries migration obligations
mig = iface.transition(cid, "migrating", evidence_ref="consumer migration")
assert mig["success"], mig.get("errors")
sup = iface.transition(cid, "superseding", evidence_ref="old path retired")
assert sup["success"], sup.get("errors")
# the OLD path record survives in the governed store (history, not
# deletion): supersession is a lifecycle state, not erasure
doc = iface.scaffold.load()
assert ss.entity(doc, "module:v1_api") is not None
assert ss.entity(doc, "consumer:dashboard") is not None
# ── 15T: failure / recovery matrix ──────────────────────────────────────────
def test_15t_failure_classification_matrix():
"""15T: every failure class classifies deterministically, with an
explicit retry budget — and PERMANENT/APPROVAL never auto-retry."""
from rune import failure_recovery as fr
cases = [
("connection reset by peer", fr.TRANSIENT, 2),
("rate limit 429 exceeded", fr.TRANSIENT, 2),
("test is flaky, occasionally fails", fr.INTERMITTENT, 3),
("schema-invalid observation payload", fr.PERMANENT, 0),
("module not found: foo", fr.PERMANENT, 0),
("user confirmation required to proceed", fr.APPROVAL_REQUIRED, 0),
("destructive external change", fr.APPROVAL_REQUIRED, 0),
("stale document committed over persisted state", fr.KNOWN_PROCESS_DEFECT, 1),
("totally novel failure mode", fr.UNKNOWN, 1),
]
for text, expected_class, expected_budget in cases:
r = fr.classify_failure(text)
assert r["class"] == expected_class, f"{text!r} → {r['class']}"
max_retries, action = r["policy"]
assert max_retries == expected_budget, text
# no infinite loops: every policy has an explicit budget
for cls in fr.CLASSES:
budget, _ = fr.POLICY[cls]
assert isinstance(budget, int) and budget <= 3
# PERMANENT and APPROVAL never retry
assert fr.POLICY[fr.PERMANENT][0] == 0
assert fr.POLICY[fr.APPROVAL_REQUIRED][0] == 0
def test_15t_known_defect_context_wins_over_markers():
from rune import failure_recovery as fr
r = fr.classify_failure("timeout during deploy",
context={"known_defect_id": "process-defect:x"})
assert r["class"] == fr.KNOWN_PROCESS_DEFECT
assert "process-defect:x" in r["evidence"]
# ── 15Y: adversarial default-mechanism matrix (remaining gaps) ──────────────
def test_15y_event_for_different_project_matches_nothing():
"""15Y case 8: an event for a different project cannot mutate the
target wait — correlation is (provider, event_type, run_ref)."""
import sys
sys.path.insert(0, str(ROOT / "src"))
from rune import event_wait as ew
import tempfile
from pathlib import Path
tmp = Path(tempfile.mkdtemp())
import os
os.environ["RUNE_WR_ROOT"] = str(tmp)
os.environ["RUNE_WR_CODE"] = "/home/nick/.local/share/mimo-wait-resume"
ew.register_process_wait("op-target", project="project-a",
watchdog_secs=60)
ew.register_process_wait("op-other", project="project-b",
watchdog_secs=60)
import importlib
sys.path.insert(0, "/home/nick/.local/share/mimo-wait-resume")
import waitstore
waitstore = importlib.reload(waitstore)
# the target's correlation is op-target; an event for op-other has a
# different run_ref → match() rejects it for the target's wait
target = None
for rec in waitstore.all_waits():
if rec.get("ext", {}).get("operation_id") == "op-target":
target = rec
assert target is not None
foreign_event = {"provider": "builder", "event_type": "builder_exit",
"run_ref": "op-other", "exit_code": 0}
assert waitstore.match(target, foreign_event) is False
own_event = {"provider": "builder", "event_type": "builder_exit",
"run_ref": "op-target", "exit_code": 0}
assert waitstore.match(target, own_event) is True
def test_15y_user_approval_gate_blocks_consequential_action():
"""15Y case 22: a consequential scope change without user authority
is refused — amend_scope is user-gated, not builder-gated."""
from rune import construction_interface as ci
import tempfile
from pathlib import Path
tmp = Path(tempfile.mkdtemp())
iface = ci.ConstructionInterface(builder="rogue-builder",
project_store=str(tmp / "s"),
project_id="t15y")
iface.scaffold.init(purpose="15y probe")
doc = iface.scaffold.load()
from rune import structural_store as ss
ss.add_entity(doc, id="contract:spec-15y", kind="contract",
identities={"objective": "spec"}, evidence_refs=["e"])
ss.add_entity(doc, id="capability:planned", kind="capability",
maturity="DECLARED", evidence_refs=["e"])
ss.add_relationship(doc, type="implements",
from_id="capability:planned",
to_id="contract:spec-15y", evidence_refs=["e"])
iface.scaffold.commit(doc, what="seed 15y spec")
sub = iface.submit_need(
"build the 15y module",
constraints={"environment": {"os": "linux"},
"contracts": ["contract:spec-15y"]},
evidence_refs=["e"], approval_context="15y")
assert sub["success"]
cid = sub["ids"]["contract_id"]
# builder attempts to broaden its own scope — refused
r = iface.amend_scope(cid, ["extra/secret.py"], reason="builder wants",
authority="builder")
assert not r["success"]
# user authority works
r2 = iface.amend_scope(cid, ["extra/needed.py"], reason="user asked",
authority="user")
assert r2["success"], r2.get("errors")