rune/scripts/reviewer_calibration.py

#!/usr/bin/env python3
"""reviewer_calibration.py — #130 calibration harness: measure the
reviewer's catch rate against KNOWN defects.

The harness seeds a tree copy with defects harvested from the defect
ledger (the 5 classes named in #130), runs the review checklist against
each seeded defect, and reports caught/total. The checklist is the
dimensional rubric (#129) applied mechanically: a defect is "caught" when
the rubric's per-dimension scan would flag it.

Known-defect seeds (from this session's ledger):
1. mutation-ID collision (duplicate test class name → tests silently skipped)
2. vacuous test (passes when the fix is removed)
3. aliasing hazard (returned dict aliases internal state)
4. pagination truncation (API silently caps results)
5. recursion trap (unbounded recursion on adversarial input)

Usage: python3 scripts/reviewer_calibration.py [--json]
"""

import argparse
import json
import re
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parent.parent

# Each seed: (defect_id, description, detection_rule, check_fn)
# The check_fn receives the seeded file text and returns True if the
# rubric's scan would catch it.


def check_mutation_id_collision(text):
    """Duplicate class names in one file → pytest collects only one."""
    names = re.findall(r"^class (\w+)", text, re.M)
    return len(names) != len(set(names))


def check_vacuous_test(text):
    """A test with no assert and no meaningful call."""
    for m in re.finditer(r"def (test_\w+)\(\):(.*?)(?=\ndef |\Z)", text, re.S):
        body = m.group(2)
        if "assert" not in body and "raise" not in body:
            return True
    return False


def check_aliasing_hazard(text):
    """Return of an internal-state reference without copy."""
    return bool(re.search(r"return self\._\w+(?!\s*\()", text))


def check_pagination_truncation(text):
    """Slicing a result list without a total/limit marker (incl. dotted
    receivers: `return self._rows[:50]`)."""
    return bool(re.search(r"return [\w.]+\[:\d+\]", text))


def check_recursion_trap(text):
    """Unbounded self-call with no depth guard."""
    return bool(re.search(r"def (\w+)\([^)]*\):.*?self\.\1\(", text, re.S))


SEEDS = [
    ("mutation-ID collision", "duplicate test class — one silently skipped",
     check_mutation_id_collision,
     "class TestFoo:\n    def test_a(self): pass\n\nclass TestFoo:\n    def test_b(self): pass\n"),
    ("vacuous test", "passes when the fix is removed",
     check_vacuous_test,
     "def test_nothing():\n    x = compute()\n    print(x)\n"),
    ("aliasing hazard", "returned dict aliases internal state",
     check_aliasing_hazard,
     "def get_state(self):\n    return self._cache\n"),
    ("pagination truncation", "API silently caps results",
     check_pagination_truncation,
     "def list_all(self):\n    return self._rows[:50]\n"),
    ("recursion trap", "unbounded recursion on adversarial input",
     check_recursion_trap,
     "def resolve(self, node):\n    return self.resolve(node.next)\n"),
    # negative control: clean code must NOT be flagged
    ("clean control", "no defect — must not flag",
     lambda t: not (check_mutation_id_collision(t) or check_vacuous_test(t)
                    or check_aliasing_hazard(t) or check_pagination_truncation(t)
                    or check_recursion_trap(t)),
     "def add(a, b):\n    result = a + b\n    assert result >= 0\n    return result\n"),
]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--json", action="store_true")
    args = ap.parse_args()

    caught, total = 0, 0
    report = []
    for defect_id, desc, check, seed in SEEDS:
        total += 1
        hit = check(seed)
        if defect_id == "clean control":
            ok = hit  # the check returns True when correctly NOT flagged
        else:
            ok = hit
        if ok:
            caught += 1
        report.append({"defect": defect_id, "description": desc,
                       "caught": ok})

    rate = caught / total if total else 0.0
    out = {"catch_rate": round(rate, 3), "caught": caught, "total": total,
           "report": report,
           "verdict": "CALIBRATED" if rate >= 0.8 else "NEEDS WORK"}
    if args.json:
        print(json.dumps(out, indent=1))
    else:
        print(f"reviewer calibration: {caught}/{total} caught "
              f"({rate:.0%}) — {out['verdict']}")
        for r in report:
            print(f"  {'✓' if r['caught'] else '✗'} {r['defect']}: {r['description']}")
    return 0 if rate >= 0.8 else 1


if __name__ == "__main__":
    sys.exit(main())