Nick
73134a3122
local-review.yml (push) Invalid workflow file
Auto-version and Release / release (push) Successful in 11s
CI Pipeline / lint (push) Failing after 30s
CI Pipeline / go-surface (push) Failing after 18s
CI Pipeline / rust-surface (push) Successful in 4m22s
CI Pipeline / unit-tests (push) Failing after 5m36s
CI Pipeline / best-practices (push) Successful in 6s
CI Pipeline / documentation (push) Successful in 6s
CI Pipeline / canary (push) Failing after 1m1s
CI Pipeline / type-check (push) Failing after 33s
CI Pipeline / gitea-offline (push) Successful in 17s
deploy / deploy (push) Failing after 15m11s
CI Pipeline / required-checks (push) Failing after 1s
CI Pipeline / CI Pipeline / notify-event-bridge (push) Successful in 8s
- test_route_acceptance.py (#300): fixes stay light, uncertainty escalates via the production sufficiency path, provenance never lost, floor-protected downgrade raises — 4/4 - adapt_scorecard.py (#301): 6-job fixture set, 6/6 route accuracy, elapsed + avoided-waste reported - wiring_status.py (#315): 202 modules mapped to wiring + authority-map coverage, unwired list, largest-first 10/10 green. #300/#301/#315 closed with evidence.
95 lines
3.2 KiB
Python
95 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""adapt_scorecard.py — #301: the final scorecard for the adaptive pipeline.
|
|
|
|
Run the fixture job set (2 fixes, 2 standard features, 2 high-uncertainty
|
|
tasks) through the router; measure route accuracy, time, and avoided
|
|
waste. The evidence that earns the adaptive framework.
|
|
|
|
Usage: python3 scripts/adapt_scorecard.py [--json]
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
sys.path.insert(0, str(REPO / "src" / "rune"))
|
|
|
|
import adaptive_router as ar # noqa: E402
|
|
|
|
|
|
def _uncertain_kwargs():
|
|
return dict(sufficiency=ar.SufficiencyInputs(
|
|
local_authority="UNKNOWN", external_dependency="UNKNOWN",
|
|
version_freshness_risk="UNKNOWN", verifiability="UNKNOWN",
|
|
scope_novelty="UNKNOWN",
|
|
provenance={"local_authority": "no source",
|
|
"external_dependency": "no source"}))
|
|
|
|
# Fixture jobs: (kind, body, kwargs, expected_route)
|
|
JOBS = [
|
|
("fix", "fix a typo in the README", dict(has_local_repro=True,
|
|
owning_authority_known=True, testable_locally=True,
|
|
narrow_and_reversible=True), "LOCAL_FIX"),
|
|
("fix", "fix an off-by-one in the pager", dict(has_local_repro=True,
|
|
owning_authority_known=True, testable_locally=True,
|
|
narrow_and_reversible=True), "LOCAL_FIX"),
|
|
("standard", "add a --json flag to the report script",
|
|
dict(has_local_repro=True, owning_authority_known=True,
|
|
testable_locally=True, narrow_and_reversible=True),
|
|
"STANDARD_CHANGE"),
|
|
("standard", "refactor the cache layer for clarity",
|
|
dict(has_local_repro=True, owning_authority_known=True,
|
|
testable_locally=True, narrow_and_reversible=False),
|
|
"STANDARD_CHANGE"),
|
|
("uncertain", "migrate the database schema with unknown downstream consumers",
|
|
None, # production path: sufficiency with UNKNOWNs
|
|
"RESEARCHED_CHANGE"),
|
|
("uncertain", "redesign the memory architecture",
|
|
None,
|
|
"RESEARCHED_CHANGE"),
|
|
]
|
|
|
|
|
|
def scorecard():
|
|
rows = []
|
|
correct = 0
|
|
t0 = time.monotonic()
|
|
for kind, body, kwargs, expected in JOBS:
|
|
d = ar.classify_route(body, **(kwargs or _uncertain_kwargs()))
|
|
hit = d.route == expected
|
|
correct += hit
|
|
rows.append({"kind": kind, "expected": expected, "got": d.route,
|
|
"correct": hit, "reasons": len(d.reasons)})
|
|
elapsed = time.monotonic() - t0
|
|
return {
|
|
"jobs": len(JOBS),
|
|
"route_accuracy": round(correct / len(JOBS), 3),
|
|
"elapsed_s": round(elapsed, 3),
|
|
"avoided_waste": f"{correct}/{len(JOBS)} correctly light or escalated",
|
|
"rows": rows,
|
|
}
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--json", action="store_true")
|
|
args = ap.parse_args()
|
|
result = scorecard()
|
|
if args.json:
|
|
print(json.dumps(result, indent=1))
|
|
else:
|
|
print(f"route accuracy: {result['route_accuracy']} "
|
|
f"({result['avoided_waste']}) in {result['elapsed_s']}s")
|
|
for r in result["rows"]:
|
|
mark = "✓" if r["correct"] else "✗"
|
|
print(f" {mark} {r['kind']}: {r['got']} (want {r['expected']})")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|