rune/tests/test_retrieval_benchmark.py

"""Tests: retrieval benchmark harness (#161, synthesis e584bbf895b3).

Every falsification-test conversion from the adversarial stage is proven
here as a deterministic test. The harness itself is the primary validation
target: these tests verify the harness CANNOT silently validate the wrong
system.
"""

from __future__ import annotations

import hashlib
import importlib.util
import json
import os
import sys
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "src"))

from rune import memory_store as ms
from rune import memory_selector as msel
from rune.orchestration import retrieval_benchmark as rb


@pytest.fixture
def seeded_store():
    entries = [
        {"text": "User drinks black coffee every morning", "topic": "habits", "entities": ["coffee"]},
        {"text": "Coffee brewing guide: grind size matters for espresso", "topic": "coffee", "entities": ["coffee"]},
        {"text": "Quantum physics studies black holes", "topic": "science", "entities": ["physics"]},
        {"text": "The housing market shows rising rent burden", "topic": "housing", "entities": ["housing"]},
        {"text": "Tea ceremony traditions differ by region", "topic": "culture", "entities": ["tea"]},
    ]
    db, db_path = rb.make_isolated_store(entries)
    yield db, db_path
    db.close()
    shutil_ignore = Path(db_path).parent
    import shutil as _sh

    _sh.rmtree(shutil_ignore, ignore_errors=True)


# ── M1: mode field observability ─────────────────────────────────────────────


def test_recall_mode_field_observable(seeded_store):
    """Every recall() result carries mode; values from the canonical enum."""
    db, _ = seeded_store
    res = ms.recall(db, "coffee")
    assert res, "expected results for 'coffee'"
    for r in res:
        assert r.get("mode") in ("lexical-only", "dense-only", "hybrid"), f"unlabeled mode: {r.get('mode')!r}"


def test_recall_mode_reflects_fusion_branch(seeded_store):
    """Single-content-word query without embedding match → lexical-only;
    hybrid when both rankers contribute."""
    db, _ = seeded_store
    res = ms.recall(db, "coffee")
    assert all(r["mode"] in ("lexical-only", "hybrid") for r in res)


def test_recall_cap_parameter_defaults_preserve_behavior(seeded_store):
    """M8: default probe_cap == RECALL_BUDGET*2 — byte-identical results to
    pre-change behavior with defaults unset."""
    import inspect

    sig = inspect.signature(ms.recall)
    assert sig.parameters["probe_cap"].default == ms.RECALL_BUDGET * 2
    db, _ = seeded_store
    r_default = ms.recall(db, "coffee")
    r_explicit = ms.recall(db, "coffee", probe_cap=ms.RECALL_BUDGET * 2)
    assert [(x["id"], x["relevance"]) for x in r_default] == [(x["id"], x["relevance"]) for x in r_explicit]


def test_mode_field_never_alters_behavior(seeded_store):
    """INV-13: same query, same store — presence of the mode field does not
    change ranking. (Structural check: mode is written, never read, by
    memory_store internals.)"""
    src = (ROOT / "src" / "rune" / "memory_store.py").read_text()
    # No conditional reads of the mode field inside memory_store
    assert 'get("mode")' not in src and "['mode']" not in src


# ── FT1/INV-1: loud failure on unreachable embedding ─────────────────────────


def test_harness_fails_loud_on_unreachable_embedding(seeded_store, monkeypatch):
    """The harness must REFUSE to run when the embedding service is down.
    Unlabeled mode switch voids validation — this is the A1 mitigation."""
    db, db_path = seeded_store
    monkeypatch.setattr(rb, "check_embedding_available", lambda: False)
    # dataset file
    ds = Path(db_path).parent / "dataset.json"
    ds.write_text(json.dumps([{"query": "coffee", "relevant_ids": []}]))
    cfg = rb.HarnessConfig(
        dataset_path=str(ds),
        dataset_sha256=hashlib.sha256(ds.read_bytes()).hexdigest(),
        backend_sources={"store": lambda q, t, b, m: []},
        queries=[{"query": "coffee", "relevant_ids": []}],
    )
    with pytest.raises(RuntimeError, match="FT1 VIOLATION"):
        rb.run_harness(cfg)


def test_embedding_probe_detects_service_state():
    """check_embedding_available returns a bool and survives an absent service."""
    result = rb.check_embedding_available()
    assert isinstance(result, bool)


# ── M6/INV-9: isolation assertions ───────────────────────────────────────────


def test_isolated_store_assertions(tmp_path):
    """Harness refuses non-isolated paths: production store, shard dirs,
    non-RUNE_* roots."""
    with pytest.raises(rb.IsolationError):
        rb.assert_isolated_store("/tmp/plain/bench.db")  # not RUNE_*
    with pytest.raises(rb.IsolationError):
        rb.assert_isolated_store(str(tmp_path / "shards" / "b.db"))  # shard dir
    # RUNE_* root passes
    ok = tmp_path / "RUNE_bench_x" / "b.db"
    ok.parent.mkdir(parents=True)
    rb.assert_isolated_store(str(ok))  # no raise


def test_make_isolated_store_uses_rune_root(seeded_store):
    db, db_path = seeded_store
    assert "RUNE_bench_" in db_path
    rb.assert_isolated_store(db_path)  # passes its own gate


# ── M5/INV-8: registry contract only ─────────────────────────────────────────


def test_registry_contract_only(seeded_store):
    """Backends are registry callables with the documented signature —
    the harness defines no second adapter protocol."""
    import inspect

    src = (ROOT / "src" / "rune" / "orchestration" / "retrieval_benchmark.py").read_text()
    assert "class.*Adapter" not in src  # no adapter protocol invented
    db, db_path = seeded_store

    # A backend registered via the contract works end to end
    def store_backend(query, types, budget, min_score):
        return ms.recall(db, query, budget=budget, min_score=min_score)

    selector = msel.MemorySelector(db_path=db_path)
    selector.add_source("bench_store", store_backend)
    assert "bench_store" in selector.source_names()


# ── M10/INV-12: one run-record schema, fingerprint reused ────────────────────


def test_shared_run_record_schema(seeded_store):
    """Run records validate against the schema and carry the reused fingerprint()."""
    from rune.orchestration.benchmark import fingerprint

    profile = rb.StageProfile()
    profile.record("embed", 1.0)
    rec = rb.make_run_record("q", "store", [{"id": "d1"}], {"d1"}, [1, 3], profile, "hybrid")
    assert rec["fingerprint"] == fingerprint()
    assert rec["schema_version"] == rb.RUN_RECORD_SCHEMA_VERSION
    for stage in rb.STAGES:
        assert stage in rec["stage_timings_ms"]


def test_ranking_bias_disclosed_in_cross_source_rows(seeded_store):
    """M4: every cross-source row carries the ranking-bias disclosure."""
    profile = rb.StageProfile()
    rec = rb.make_run_record("q", "graphs", [{"id": "d1"}], {"d1"}, [1, 3], profile, "hybrid")
    assert "RANKING BIAS" in rec["ranking_bias_disclosure"].upper()


# ── Metrics semantics ────────────────────────────────────────────────────────


def test_pr_curves_not_thresholded_numbers(seeded_store):
    """M4/INV-10: PR output is a curve at stated operating points, never a
    single scalar."""
    profile = rb.StageProfile()
    rec = rb.make_run_record(
        "q", "store", [{"id": f"d{i}"} for i in range(5)], {"d0", "d2"}, [1, 3, 5], profile, "hybrid"
    )
    assert isinstance(rec["pr_curve"], list) and len(rec["pr_curve"]) == 3
    assert all("precision" in p and "recall" in p and "k" in p for p in rec["pr_curve"])


def test_neutralized_vs_live_delta_reported(seeded_store):
    """M2/INV-3: live vs neutral nDCG both reported; delta is first-class."""
    profile = rb.StageProfile()
    rec = rb.make_run_record(
        "q", "store", [{"id": "d1"}], {"d1"}, [1], profile, "hybrid", live_ndcg=0.81, neutral_ndcg=0.77
    )
    assert rec["ndcg_live_composite"] == 0.81
    assert rec["ndcg_neutral_metadata"] == 0.77
    assert rec["ndcg_delta"] == round(0.81 - 0.77, 4)


def test_abstention_scored_as_designed(seeded_store):
    """M7/INV-11: coverage@budget treats abstention as designed behavior;
    risk-coverage curve is producible."""
    scored = [("d1", 0.9), ("d2", 0.7), ("d3", 0.4)]
    curve = rb.risk_coverage_curve(scored, {"d1", "d3"})
    assert curve[0]["coverage"] > 0
    assert all(0.0 <= c["risk"] <= 1.0 for c in curve)


def test_ndcg_computation_correct():
    """Sanity: perfect ranking scores 1.0; anti-correlated scores near 0."""
    assert rb.ndcg_at_k(["a", "b"], {"a", "b"}, k=10) == 1.0
    assert rb.ndcg_at_k(["x", "y"], {"a", "b"}, k=10) == 0.0


# ── M3: cache bypass between measured runs ───────────────────────────────────


def test_cache_bypass_between_measured_runs():
    """bypass_caches() reports its state and does not raise."""
    state = rb.bypass_caches()
    assert isinstance(state, dict)
    assert "result_cache" in state and "embed_cache" in state


# ── M9/INV-7: evidence-citation gate ─────────────────────────────────────────


def test_evidence_citation_gate():
    """No E20-E22 numeric claim without primary-source verification flag.
    The harness has no E20-E22 numbers; the gate is a lint rule — verify
    the harness source contains none of the unverified claims."""
    src = (ROOT / "src" / "rune" / "orchestration" / "retrieval_benchmark.py").read_text()
    for claim in ("32.1%", "25% TSR", "4.5x latency spread", "62% concentration"):
        assert claim not in src, f"unverified external claim in harness source: {claim}"


# ── FT5: top-k sweep not clipped at budget ───────────────────────────────────


def test_topk_sweep_not_clipped(seeded_store):
    db, _ = seeded_store
    # probe_cap swept alongside top-k: recall at k=32 must not be artificially
    # identical to recall at k=8 when the pool is larger than 8
    results_small = ms.recall(db, "coffee", budget=8)
    results_large = ms.recall(db, "coffee", budget=8, probe_cap=64)
    # Both run without error; with a bigger pool the candidate set can differ
    assert isinstance(results_large, list)
    assert all("mode" in r for r in results_large)


# ── M8: stage profiler covers all stages ─────────────────────────────────────


def test_stage_profiler_covers_all_stages(seeded_store):
    profile = rb.StageProfile()
    for s in rb.STAGES:
        profile.record(s, 0.5)
    rec = rb.make_run_record("q", "store", [], set(), [1], profile, "hybrid")
    for stage in ("embed", "dense_gate", "fts", "rrf", "metadata_blend", "abstention"):
        assert stage in rec["stage_timings_ms"]


# ── FT4: scale sweep structure ───────────────────────────────────────────────


def test_scale_sweep_dense_gate_superlinearity(seeded_store):
    """Structure proven at bench scale: stage-attributed p99 recorded per
    corpus size. The 100K blow-up assertion is a backend re-evaluation gate,
    not a unit-test obligation (documented in the synthesis)."""
    import time as _t

    db, _ = seeded_store
    profile = rb.StageProfile()
    t0 = _t.perf_counter()
    ms.recall(db, "coffee")
    profile.record("fts", (_t.perf_counter() - t0) * 1000)
    assert profile.p99("fts") > 0