rune/tests/test_evidence_probe.py

#!/usr/bin/env python3
"""tests/test_evidence_probe.py — Targeted Evidence Probe canaries (§20.D).

Canary 2: otherwise-local task → material external unknown → probe GROUNDED
          → returns to implementation path, does NOT launch full research.
Canary 3: probe STILL_UNKNOWN / CONTRADICTED → recorded escalation →
          RESEARCHED_CHANGE (no probe loops).
Plus module gates: schema validation, budget (no recursion), file authority.
"""

import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

PKG_ROOT = os.path.join(os.path.dirname(__file__), "..")
sys.path.insert(0, os.path.join(PKG_ROOT, "src"))
sys.path.insert(0, os.path.join(PKG_ROOT, "scripts"))

from rune import adaptive_router  # noqa: E402
from rune import evidence_probe  # noqa: E402
from research_stage_executor import StageExecutor, make_event_id  # noqa: E402


def _issue_event(issue_no, stage="ISSUE", attempt=1):
    return {
        "event_type": "ResearchStageReady",
        "event_id": make_event_id(issue_no, f"research-{issue_no}", 1, stage, attempt),
        "issue": issue_no,
        "round": 1,
        "stage": stage,
        "attempt": attempt,
    }


# An otherwise-local task with ONE material external unknown: the issue
# references an external library's behaviour but is a known-pattern change.
AMBIGUOUS_ISSUE = {
    "number": 9100,
    "title": "Handle platformdirs config-dir resolution on new installs",
    "body": (
        "Wire the config dir through the platformdirs library package. "
        "Reference: https://platformdirs.readthedocs.io. Depends on "
        "current behaviour of the library's version. Existing test: "
        "tests/test_config_paths.py covers the local wiring."
    ),
    "labels": [],
}

GROUNDED_PROBE = {
    "question": "probe",
    "source": "https://platformdirs.readthedocs.io/changelog.html",
    "answer": "Current version guarantees the assumed behaviour.",
    "implication": "Local wiring is sufficient; no version guard needed.",
    "status": "GROUNDED",
}

UNKNOWN_PROBE = {
    "question": "probe",
    "source": "",
    "answer": "Could not find authoritative evidence.",
    "implication": "Uncertainty remains.",
    "status": "STILL_UNKNOWN",
}


def _runner_returning(payload):
    def runner(prompt, artifact_path):
        with open(artifact_path, "w") as f:
            json.dump(dict(payload, question=payload.get("question", "probe")), f)

    return runner


def _make_executor(tmpdir, issue_data, runner):
    ex = StageExecutor(issue=issue_data["number"], artifact_dir=tmpdir, round_no=1, probe_runner=runner)
    ex._fetch_issue = lambda: issue_data
    return ex


class Canary2ProbeGroundedReturnsToLocal(unittest.TestCase):
    def test_grounded_probe_resolves_unknown_without_deep_research(self):
        with tempfile.TemporaryDirectory() as tmp:

            def runner(prompt, artifact_path):
                # The prompt must name the EXACT artifact path (ADAPT-9)
                self.assertIn(artifact_path, prompt)
                with open(artifact_path, "w") as f:
                    json.dump(dict(GROUNDED_PROBE), f)

            ex = _make_executor(tmp, AMBIGUOUS_ISSUE, runner)
            result = ex.execute(_issue_event(9100))

            # The route must NOT be deep research: the probe grounded the
            # unknown and the task returns to the local implementation path.
            self.assertEqual(
                result["status"], "ROUTED_LOCAL_FIX", f"grounded probe should return to local lane: {result}"
            )
            # Exactly ONE probe ran — no loops.
            self.assertEqual(len(result["probes"]), 1)
            self.assertEqual(result["probes"][0]["status"], "GROUNDED")
            # Probe persisted with citations
            probe = json.loads((Path(tmp) / "probe.json").read_text())
            self.assertEqual(probe["status"], "GROUNDED")
            self.assertIn("https://", probe["source"])
            # Route log: INITIAL_ROUTE (+ DOWNGRADE when the probe resolved local
            # authority — §7: transitions persisted with reasons).
            events = [json.loads(l) for l in (Path(tmp) / "route.jsonl").read_text().strip().splitlines()]
            self.assertEqual(events[0]["kind"], "INITIAL_ROUTE")
            self.assertEqual(events[-1]["route"], "LOCAL_FIX")
            kinds = [e["kind"] for e in events]
            if "DOWNGRADE" in kinds:
                dg = next(e for e in events if e["kind"] == "DOWNGRADE")
                self.assertEqual(dg["trigger"], "probe_grounded_local_authority")
                self.assertEqual(dg["previous_route"], "STANDARD_CHANGE")
            self.assertIn("evidence probe GROUNDED", events[0].get("provenance", {}).get("local_authority", ""))


class Canary3UnresolvedEscalates(unittest.TestCase):
    def test_still_unknown_escalates_to_researched_change(self):
        with tempfile.TemporaryDirectory() as tmp:
            ex = _make_executor(tmp, AMBIGUOUS_ISSUE, _runner_returning(dict(UNKNOWN_PROBE)))
            result = ex.execute(_issue_event(9100))

            # Escalation: enters the canonical chain (no terminal local route)
            self.assertNotEqual(result.get("status"), "ROUTED_LOCAL_FIX")
            events = [json.loads(l) for l in (Path(tmp) / "route.jsonl").read_text().strip().splitlines()]
            kinds = [e["kind"] for e in events]
            self.assertIn("ESCALATION", kinds)
            esc = next(e for e in events if e["kind"] == "ESCALATION")
            self.assertEqual(esc["new_route"], "RESEARCHED_CHANGE")
            self.assertEqual(esc["trigger"], "evidence_probe_unresolved")
            self.assertIn("STILL_UNKNOWN", esc["evidence"])
            # Transition identity fields present
            for field in ("previous_route", "trigger", "evidence"):
                self.assertIn(field, esc)
            # Probe recorded the failure honestly — in the PERSISTED state
            # (the escalation path returns into the chain; the durable
            # artifact is the tree, not the ephemeral return value).
            self.assertEqual(ex.tree.state["probes"][0]["status"], "STILL_UNKNOWN")
            probe = json.loads((Path(tmp) / "probe.json").read_text())
            self.assertEqual(probe["status"], "STILL_UNKNOWN")

    def test_no_probe_recursion_budget(self):
        with tempfile.TemporaryDirectory() as tmp:
            for i in range(evidence_probe.MAX_PROBES_PER_TASK):
                evidence_probe.run_probe(f"unknown {i}", tmp, runner=_runner_returning(dict(UNKNOWN_PROBE)))
            with self.assertRaises(evidence_probe.ProbeBudgetExceeded):
                evidence_probe.run_probe("one too many", tmp, runner=_runner_returning(dict(UNKNOWN_PROBE)))


class ProbeModuleGates(unittest.TestCase):
    def test_schema_validation(self):
        bad = {"question": "q", "source": "s", "answer": "a", "implication": "i", "status": "MAYBE"}
        errors = evidence_probe.validate_probe(bad)
        self.assertTrue(any("status" in e for e in errors))
        grounded_no_source = dict(GROUNDED_PROBE, source="x")
        errors = evidence_probe.validate_probe(grounded_no_source)
        self.assertTrue(any("source" in e for e in errors))
        self.assertEqual(evidence_probe.validate_probe(dict(GROUNDED_PROBE)), [])

    def test_missing_artifact_fails_loud(self):
        with tempfile.TemporaryDirectory() as tmp:

            def silent_runner(prompt, artifact_path):
                pass  # worker wrote nothing

            with self.assertRaises(ValueError) as ctx:
                evidence_probe.run_probe("q", tmp, runner=silent_runner)
            self.assertIn("declared artifact", str(ctx.exception))


if __name__ == "__main__":
    unittest.main()