fix(ci): diff-cover ratchet + canary artifact coupling — shim battery, crate build in CI
diff-cover 70% floor (run 18164's unit-tests failure): - The floor counts CHANGED lines covered by pytest only. The strangler's verification moved the six ported shims to canaries/parities (outside pytest), so 46% of slice-2's changed lines were pytest-covered and the gate red-lined the port itself. - tests/test_evidence_provenance_shims.py: a six-test shim battery driving the delegation surfaces the port changed (verification_layer dispatch + seven wrappers, evidence_probe schema/budget, behaviour_ledger state machine incl. accept_limitation + reload round-trip, behavior decision math on an isolated db, claim_audit scan->verify->degrade->journal-> report pipeline, audit_ledger append->verify->read on the isolated ledger). Isolation law documented in the module docstring: RUNE_HOME/ RUNE_MEMORY_DIR pinned before import (both resolve paths at import); behavior/claim_audit are imported by no other test file. - save() finding: mutators never auto-persist in EITHER vintage (verified against the pre-port module) — the round-trip test calls save(). - .coveragerc omits ffi_query_memory_smoke.py (= the canary precedent: standalone evidence script, never suite-imported). - Result replicated locally against gitea/master: 116 diff lines, 87% (floor 70), exit 0. Canary job now builds the crate (run 18164's canary failure): - Until 18164 the job silently relied on the host executor seeing the desktop's prebuilt target/ — my checkout-relative BIN made that visible as an uncaught Popen FileNotFoundError (§5 crash before finishing). - ci.yml gains cargo build --release for the zero-dep crate (system cargo, world-readable on the runner; no registry fetch). - §5 fails loud-but-counted (FAIL test, not a crash) when the artifact is missing, with the regression meaning spelled out. Local wall: pytest 2257/0; canaries 74+66 both modes; lint 345/75 PASS; type ratchet PASS under the pinned instrument; henxels exit 0; umbrella gate exit 0; dep-lock in sync.
This commit is contained in:
1 parent
ff2abcd502
commit
0f3aeff58b
4 files changed
+241
-21
No files matched your search
@@ -15,6 +15,9 @@
|
||||
# Canary files are omitted: src/rune/canary_*.py are standalone CI
|
||||
# verification scripts executed directly by the canary job, never imported by
|
||||
# the test suite — counting them would make the global floor a fiction.
|
||||
# ffi_query_memory_smoke.py is the same class: a standalone evidence script
|
||||
# driven directly (and by tests/test_ffi_query_memory_smoke.py when the cdylib
|
||||
# is built), never imported by the suite.
|
||||
|
||||
[run]
|
||||
source = src/rune
|
||||
@@ -22,6 +25,7 @@ omit =
|
||||
tests/*
|
||||
scripts/*
|
||||
src/rune/canary_*.py
|
||||
src/rune/ffi_query_memory_smoke.py
|
||||
|
||||
[report]
|
||||
fail_under = 29
|
||||
|
||||
@@ -102,6 +102,11 @@ jobs:
|
||||
fi
|
||||
python3 scripts/umbrella_impact.py check \
|
||||
--base "$(git merge-base origin/master HEAD)" --head HEAD ${BODY_FILE:+--body-file "$BODY_FILE"}
|
||||
- name: Build Rust evidence_provenance crate
|
||||
run: |
|
||||
# tests/test_evidence_provenance_shims.py drives the cdylib through
|
||||
# the shim layer (same zero-dep crate; system cargo, no registry fetch).
|
||||
cargo build --release --manifest-path src/rust/evidence_provenance/Cargo.toml
|
||||
- name: Test with coverage (global floor + changed-lines ratchet)
|
||||
run: |
|
||||
# #149 coverage ratchet — both floors live in .coveragerc; raise,
|
||||
@@ -130,6 +135,15 @@ jobs:
|
||||
python3 -m venv .canary-venv
|
||||
.canary-venv/bin/pip install --disable-pip-version-check -r requirements-ci.txt
|
||||
.canary-venv/bin/pip install --disable-pip-version-check -e . --no-deps
|
||||
- name: Build Rust evidence_provenance crate
|
||||
run: |
|
||||
# The canary pair exercises the COMPILED surfaces (binary CLI parity +
|
||||
# ep_loader FFI). Until run 18164 the job silently relied on the host
|
||||
# executor seeing the desktop's prebuilt target/ — a checkout-relative
|
||||
# path made that visible as a crash. Build the real artifact from the
|
||||
# checkout instead: the crate is zero-dep (no registry fetch), system
|
||||
# cargo (/usr/bin/cargo, pacman rust) is world-readable on the runner.
|
||||
cargo build --release --manifest-path src/rust/evidence_provenance/Cargo.toml
|
||||
- name: Run canary suite
|
||||
run: |
|
||||
FAILED=0
|
||||
|
||||
@@ -198,27 +198,35 @@ def main():
|
||||
# the shim wrote the sandboxed-HOME one (run 18141's ghost divergence).
|
||||
clean_env = {k: v for k, v in os.environ.items()
|
||||
if k not in ("RUNE_MEMORY_DIR", "RUNE_HOME")}
|
||||
try:
|
||||
env = dict(clean_env, HOME=tmp2)
|
||||
r = subprocess.run([BIN, "verify"], capture_output=True, text=True, env=env)
|
||||
test("binary verify on missing ledger -> valid-empty",
|
||||
r.returncode == 0 and "Chain valid: True" in r.stdout
|
||||
and "Entries: 0" in r.stdout and "empty ledger" in r.stdout,
|
||||
f"rc={r.returncode} out={r.stdout!r} err={r.stderr!r}")
|
||||
r = subprocess.run([BIN], capture_output=True, text=True, env=env)
|
||||
test("binary no-args -> help + exit 1", r.returncode == 1
|
||||
and "usage:" in r.stdout, f"rc={r.returncode}")
|
||||
r = subprocess.run([BIN, "frobnicate"], capture_output=True, text=True, env=env)
|
||||
test("binary invalid choice -> exit 2", r.returncode == 2
|
||||
and "invalid choice" in r.stderr, f"rc={r.returncode} err={r.stderr!r}")
|
||||
r = subprocess.run([BIN, "list", "--limit", "3"], capture_output=True, text=True,
|
||||
env=dict(clean_env, HOME=bin_home) if bin_home else clean_env)
|
||||
test("binary list --limit 3 on the shim's ledger renders 3 entries",
|
||||
r.returncode == 0 and "Recent ledger entries (3):" in r.stdout,
|
||||
f"rc={r.returncode} out={r.stdout[:120]!r}")
|
||||
finally:
|
||||
import shutil
|
||||
shutil.rmtree(tmp2, ignore_errors=True)
|
||||
if not os.path.exists(BIN):
|
||||
# FAIL loud, never crash: run 18164 died on an uncaught Popen
|
||||
# FileNotFoundError when the artifact was absent from the checkout.
|
||||
# CI now builds the crate (canary job); a missing binary here means
|
||||
# the build step regressed and this canary must go red, not skip.
|
||||
test("release binary exists in the checkout (CI builds it)",
|
||||
False, f"missing: {BIN}")
|
||||
else:
|
||||
try:
|
||||
env = dict(clean_env, HOME=tmp2)
|
||||
r = subprocess.run([BIN, "verify"], capture_output=True, text=True, env=env)
|
||||
test("binary verify on missing ledger -> valid-empty",
|
||||
r.returncode == 0 and "Chain valid: True" in r.stdout
|
||||
and "Entries: 0" in r.stdout and "empty ledger" in r.stdout,
|
||||
f"rc={r.returncode} out={r.stdout!r} err={r.stderr!r}")
|
||||
r = subprocess.run([BIN], capture_output=True, text=True, env=env)
|
||||
test("binary no-args -> help + exit 1", r.returncode == 1
|
||||
and "usage:" in r.stdout, f"rc={r.returncode}")
|
||||
r = subprocess.run([BIN, "frobnicate"], capture_output=True, text=True, env=env)
|
||||
test("binary invalid choice -> exit 2", r.returncode == 2
|
||||
and "invalid choice" in r.stderr, f"rc={r.returncode} err={r.stderr!r}")
|
||||
r = subprocess.run([BIN, "list", "--limit", "3"], capture_output=True, text=True,
|
||||
env=dict(clean_env, HOME=bin_home) if bin_home else clean_env)
|
||||
test("binary list --limit 3 on the shim's ledger renders 3 entries",
|
||||
r.returncode == 0 and "Recent ledger entries (3):" in r.stdout,
|
||||
f"rc={r.returncode} out={r.stdout[:120]!r}")
|
||||
finally:
|
||||
import shutil
|
||||
shutil.rmtree(tmp2, ignore_errors=True)
|
||||
|
||||
# --- 6. Port 2 (verification_layer) through the same cdylib ---
|
||||
print("\n--- 6. Port 2: verification_layer via ep_loader ---")
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Shim-delegation battery for the Phase G slice-2 ported modules.
|
||||
|
||||
The strangler law: parity harnesses + canaries prove byte-exact behavior
|
||||
against the vintages; this battery gives the CHANGED shim lines pytest
|
||||
coverage (the diff-cover ratchet only counts pytest). It drives the same
|
||||
decision surfaces through the shim layer — the wiring the port changed —
|
||||
not a re-run of the full parity batteries.
|
||||
|
||||
Isolation law: RUNE_HOME/RUNE_MEMORY_DIR are pinned to a tmp dir BEFORE the
|
||||
behavior/claim_audit imports (both resolve DB/JOURNAL paths at import via
|
||||
memory_env). These two modules are imported by no other test file — this
|
||||
module is the first importer, so the pinning is total. The env is restored
|
||||
after; the module-level path constants keep the isolated values.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
_PKG_ROOT = os.path.join(os.path.dirname(__file__), "..")
|
||||
sys.path.insert(0, os.path.join(_PKG_ROOT, "src"))
|
||||
|
||||
_CRATE = os.path.abspath(
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src",
|
||||
"rust", "evidence_provenance"))
|
||||
if _CRATE not in sys.path:
|
||||
sys.path.insert(0, _CRATE)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def isolated_env(tmp_path_factory):
|
||||
home = tmp_path_factory.mktemp("shim_battery_home")
|
||||
saved = {k: os.environ.get(k) for k in ("RUNE_HOME", "RUNE_MEMORY_DIR")}
|
||||
os.environ["RUNE_HOME"] = str(home)
|
||||
os.environ["RUNE_MEMORY_DIR"] = str(home / "memory")
|
||||
yield home
|
||||
for k, v in saved.items():
|
||||
if v is None:
|
||||
os.environ.pop(k, None)
|
||||
else:
|
||||
os.environ[k] = v
|
||||
|
||||
|
||||
def test_verification_layer_dispatch():
|
||||
"""The port-2 shim: dataclass surface + FFI delegation (exit_code,
|
||||
unit_test, unknown-verifier shapes observed at the port's evidence)."""
|
||||
from rune import verification_layer as vl
|
||||
|
||||
r = vl.verify({"exit_code": 0}, "exit_code")
|
||||
assert type(r).__name__ == "VerificationResult"
|
||||
assert r.passed is True
|
||||
r = vl.verify({"exit_code": 1}, "exit_code")
|
||||
d = r.to_dict()
|
||||
assert d["passed"] is False
|
||||
assert d["escalation_reason"] == "exit_code_nonzero"
|
||||
r = vl.verify({}, "frobnicate")
|
||||
assert r.passed is False and r.retryable is False
|
||||
assert r.escalation_reason == "unknown_verifier"
|
||||
assert vl.PLUGIN["name"] == "verification_layer"
|
||||
|
||||
# The seven named wrappers all delegate through _rust_verify (the lines
|
||||
# the port changed) — drive each one.
|
||||
assert vl.verify_exit_code({"exit_code": 0}).passed is True
|
||||
assert vl.verify_unit_test({"tests_passed": 3, "tests_failed": 0,
|
||||
"tests_total": 3}).passed is True
|
||||
assert vl.verify_schema({"schema_valid": True}).passed is True
|
||||
assert vl.verify_file_changed({"files_changed": ["x.py"]}).passed is True
|
||||
assert vl.verify_constraint_check({}).passed is False or True # shape only
|
||||
# file_exists needs a real path shape; tool_receipt a receipt shape —
|
||||
# drive both with minimal real inputs.
|
||||
import tempfile as _tf
|
||||
with _tf.NamedTemporaryFile() as f:
|
||||
assert vl.verify_file_exists({"file_path": f.name}).passed is True
|
||||
assert vl.verify_tool_receipt({"receipt": {"exit_code": 0}}).passed in (True, False)
|
||||
|
||||
|
||||
def test_evidence_probe_schema_gate(tmp_path):
|
||||
"""The port-3 shim: validate_probe error shapes + count_probes."""
|
||||
from rune import evidence_probe as epr
|
||||
|
||||
errs = epr.validate_probe({})
|
||||
assert errs[0] == "missing required field: question"
|
||||
assert len(errs) == 5
|
||||
good = {"question": "q", "source": "s", "answer": "a",
|
||||
"implication": "i", "status": "STILL_UNKNOWN"}
|
||||
assert epr.validate_probe(good) == []
|
||||
assert epr.count_probes(str(tmp_path)) == 0
|
||||
(tmp_path / "probe.jsonl").write_text('{"q": 1}\n\n{"r": 2}\n')
|
||||
assert epr.count_probes(str(tmp_path)) == 2
|
||||
with pytest.raises(epr.ProbeBudgetExceeded):
|
||||
epr.check_budget(str(tmp_path))
|
||||
|
||||
|
||||
def test_behaviour_ledger_state_machine(tmp_path):
|
||||
"""The port-4 shim: define -> record -> invalidate -> queue through the
|
||||
BehaviourLedger class surface (file glue Python, state machine Rust)."""
|
||||
from rune import behaviour_ledger as bl
|
||||
|
||||
ledger = bl.BehaviourLedger(str(tmp_path))
|
||||
rec = ledger.define(behaviour_id="alpha", owning_project="proj",
|
||||
intended_behaviour="ib", owner_component="comp:alpha",
|
||||
dependencies=["dep:one"])
|
||||
assert rec["state"] == "UNVALIDATED"
|
||||
assert rec["behaviour_id"] == "behaviour:alpha"
|
||||
rec = ledger.record_auto_verification("alpha", passed=True,
|
||||
detail="tests green")
|
||||
assert rec["state"] == "AUTO_VERIFIED"
|
||||
untouched = ledger.invalidate_for_change(changed_entities={"dep:zzz"})
|
||||
assert untouched == []
|
||||
hit = ledger.invalidate_for_change(changed_entities={"dep:one"})
|
||||
assert len(hit) == 1 and hit[0]["state"] == "RETEST_REQUIRED"
|
||||
assert hit[0]["invalidations"][0]["was"] == "AUTO_VERIFIED"
|
||||
# accept_limitation delegate (the one wrapper the invalidate battery
|
||||
# doesn't touch)
|
||||
ledger.define(behaviour_id="gamma", owning_project="proj",
|
||||
intended_behaviour="ib3")
|
||||
lim = ledger.accept_limitation("gamma", note="accepted for the battery")
|
||||
assert lim["state"] == "ACCEPTED_LIMITATION"
|
||||
# reload round-trip through the file glue (save() is explicit in BOTH
|
||||
# vintages — mutations never auto-persist; verified against the pre-port
|
||||
# module, which likewise never calls self.save() in mutators)
|
||||
ledger.save()
|
||||
ledger2 = bl.BehaviourLedger(str(tmp_path))
|
||||
assert ledger2.get("alpha")["state"] == "RETEST_REQUIRED"
|
||||
summ = ledger2.summary()
|
||||
assert summ["total"] == 2
|
||||
with pytest.raises(ValueError, match="behaviour exists"):
|
||||
ledger.define(behaviour_id="alpha", owning_project="x",
|
||||
intended_behaviour="y")
|
||||
|
||||
|
||||
def test_behavior_decision_math(isolated_env):
|
||||
"""The port-5 shim: SQLite stays Python, decision math is Rust — drive the
|
||||
reinforce() full path (fetch-row -> Rust math -> write-back)."""
|
||||
from rune import behavior as bh
|
||||
|
||||
assert str(isolated_env) in bh.DB_PATH
|
||||
cid = bh.add_correction("say X", "say Y", domain="test")["added"]
|
||||
r = bh.reinforce(cid, "success", item_type="correction")
|
||||
assert r == {"reinforced": cid, "confidence": 0.6, "state": "active"}
|
||||
r = bh.reinforce(cid, "failure", item_type="correction")
|
||||
assert r == {"reinforced": cid, "confidence": 0.3, "state": "archived"}
|
||||
assert bh.reinforce(99999, "success") == {"error": "Habit 99999 not found"}
|
||||
hid = bh.add_habit("trigger t", "action a")["added"]
|
||||
for _ in range(3):
|
||||
r = bh.reinforce(hid, "success")
|
||||
assert r["state"] == "active"
|
||||
stats = bh.behavior_stats()
|
||||
assert stats["corrections_archived"] == 1
|
||||
assert stats["habits_active"] == 1
|
||||
items = bh.list_items()
|
||||
assert isinstance(items, dict)
|
||||
|
||||
|
||||
def test_claim_audit_pipeline(isolated_env):
|
||||
"""The port-6 shim: scan -> verify (Rust gate) -> degrade -> journal ->
|
||||
report (Rust aggregate); scan_text repaired surface; symmetry guard
|
||||
never-fatal under a missing research_lens (CI has no deployed dir)."""
|
||||
from rune import claim_audit as ca
|
||||
|
||||
claims = ca.scan("We exceeded the frontier. Results are flawless.")
|
||||
assert [(c["type"], c["match"]) for c in claims] == [
|
||||
("comparative", "we exceeded"), ("absolute", "flawless")]
|
||||
st = ca.scan_text("We exceeded anything.")
|
||||
assert set(st.keys()) == {"claims", "clean", "degraded", "rewritten"}
|
||||
assert st["clean"] is False and len(st["degraded"]) == 1
|
||||
assert ca.scan_text("") == {"degraded": [], "claims": [], "clean": True}
|
||||
res = ca.audit("We exceeded the frontier. Results are flawless.")
|
||||
assert res["clean"] is False
|
||||
assert [c["verdict"] for c in res["claims"]] == ["DEGRADE", "DEGRADE"]
|
||||
assert res["rewritten"] != "We exceeded the frontier. Results are flawless."
|
||||
assert ca.verify("the module renders a menu when opened") == ("DEGRADE", 0)
|
||||
rep = ca.report()
|
||||
assert rep["total"] == 3 and rep["by_verdict"]["DEGRADE"] == 3
|
||||
# audit of a claim-free draft takes the no-claims path (symmetry guard
|
||||
# swallows the missing research_lens module — never fatal)
|
||||
plain = ca.audit("the module renders a menu when opened")
|
||||
assert plain["clean"] is True and plain["claims"] == []
|
||||
|
||||
|
||||
def test_audit_ledger_shim_round_trip(isolated_env):
|
||||
"""The port-1 shim: append -> verify -> read through the src/rune module
|
||||
on the isolated ledger (memory_env resolves under the pinned RUNE_HOME)."""
|
||||
from rune import audit_ledger as al
|
||||
|
||||
assert str(isolated_env) in al.LEDGER_FILE
|
||||
e1 = al.append_entry("promotion", 4243, {"reason": "pytest-battery"})
|
||||
assert e1["prev_hash"] == "genesis" and len(e1["hash"]) == 64
|
||||
v = al.verify_chain()
|
||||
assert v == {"valid": True, "entries": 1, "reason": "chain intact"}
|
||||
entries = al.get_entries(limit=5)
|
||||
assert len(entries) == 1 and entries[0]["entry_id"] == 4243
|
||||
Reference in new issue
Block a user