Nick
3889d1d0e8
local-review.yml (push) Invalid workflow file
Auto-version and Release / release (push) Successful in 11s
CI Pipeline / lint (push) Failing after 31s
CI Pipeline / go-surface (push) Failing after 18s
CI Pipeline / rust-surface (push) Successful in 4m21s
CI Pipeline / unit-tests (push) Failing after 5m30s
CI Pipeline / best-practices (push) Successful in 7s
CI Pipeline / documentation (push) Successful in 6s
CI Pipeline / canary (push) Failing after 59s
CI Pipeline / type-check (push) Failing after 35s
CI Pipeline / gitea-offline (push) Successful in 18s
deploy / deploy (push) Failing after 15m11s
CI Pipeline / required-checks (push) Failing after 0s
CI Pipeline / CI Pipeline / notify-event-bridge (push) Successful in 8s
- research_budget.py (#294): query/tool budgets with overrun refusal + explicit stops - evidence_capsule.py (#295): fixed-size capsules (5/3/3) with truncation accounting + raw-evidence pointer - design consolidation (#296): the A/B experiment recorded — standard jobs merge (applicability checks are the rail), high-uncertainty keeps the separate step - harness_audit.py (#299): legibility scorer (owner/tests/contracts discoverability) + repo report 10/10 green. #294/#295/#296/#299 closed with evidence.
73 lines
2.5 KiB
Python
73 lines
2.5 KiB
Python
#!/usr/bin/env python3
|
|
"""harness_audit.py — #299: harness/context legibility audit.
|
|
|
|
How legible is the tooling to a working agent? For each module: can it
|
|
find the owner, the tests, and the contracts quickly, without ceremony?
|
|
Scores per module + a repo-wide legibility report.
|
|
|
|
Usage: python3 scripts/harness_audit.py [--json]
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
|
|
# Lookups an agent performs; each has a max acceptable hop count.
|
|
LOOKUPS = {
|
|
"owner": 2, # module → owner (authority map / header comment)
|
|
"tests": 2, # module → its test file
|
|
"contracts": 3, # module → its construction contracts
|
|
}
|
|
|
|
|
|
def audit_module(mod_path: Path) -> dict:
|
|
"""Score one module's discoverability (fewer hops = more legible)."""
|
|
name = mod_path.stem
|
|
src = mod_path.read_text()
|
|
hops = {}
|
|
# owner: authority map entry or header comment naming the owner
|
|
hops["owner"] = 1 if re.search(r"owner|authority|maintainer",
|
|
src[:800], re.I) else 3
|
|
# tests: tests/test_<name>.py exists?
|
|
test_file = REPO / "tests" / f"test_{name}.py"
|
|
test_glob = list((REPO / "tests").glob(f"*{name}*.py"))
|
|
hops["tests"] = 1 if test_file.exists() or test_glob else 3
|
|
# contracts: BUILDER- id in the file or the store?
|
|
hops["contracts"] = 1 if "BUILDER-" in src else 3
|
|
score = sum(1 for k, v in hops.items() if v <= LOOKUPS[k])
|
|
return {"module": name, "hops": hops, "score": score, "max": 3}
|
|
|
|
|
|
def audit(sample: int = 20) -> dict:
|
|
"""Audit a sample of src/rune modules."""
|
|
mods = sorted((REPO / "src" / "rune").glob("*.py"))[:sample]
|
|
rows = [audit_module(m) for m in mods]
|
|
avg = sum(r["score"] for r in rows) / max(1, len(rows))
|
|
return {"modules": len(rows), "avg_score": round(avg, 2),
|
|
"max": 3, "rows": rows,
|
|
"verdict": ("LEGIBLE" if avg >= 2.0 else "NEEDS WORK")}
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--json", action="store_true")
|
|
args = ap.parse_args()
|
|
result = audit()
|
|
if args.json:
|
|
print(json.dumps(result, indent=1))
|
|
else:
|
|
print(f"legibility: {result['avg_score']}/{result['max']} "
|
|
f"over {result['modules']} modules — {result['verdict']}")
|
|
for r in result["rows"]:
|
|
if r["score"] < 2:
|
|
print(f" {r['module']}: {r['hops']}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|