rune/tests/test_failure_auto_filer.py

"""Failure auto-filer (#122): subsystem failures become forge issues.

Core tests: signature extraction, dedup via NEW server-side search_issues,
file-once + comment-on-repeat, the ONE tier gate (T0/T1 file, T2 journal
only), and the spool contract (any client exception lands in the journal
as a failure_auto_file_spool line, nothing ever raises into the failure
path, T2 failures produce ZERO client calls).

``FailureAutoFiler.file(...)`` is the synchronous pipeline step (used by
the worker thread, reconcile, and these tests); ``submit(...)`` is the
failure-path entry that only enqueues.
"""

import json

import pytest

from rune import event_bus
from rune.failure_auto_filer import (
    FailureAutoFiler,
    JsonlSpool,
    error_class_of,
    excerpt,
    marker_for,
    redact,
    signature_of,
)
from rune.forge_client import ForgeClient, NullAdapter

MARKER = "[auto-failure:memory_selector:RuntimeError]"


class FakeJournal:
    """In-memory stand-in for the spool-journal protocol."""

    def __init__(self):
        self.lines: list[dict] = []
        self.consumed: set[str] = set()

    def append(self, record: dict) -> None:
        self.lines.append(record)

    def unconsumed(self) -> list[dict]:
        return [
            r
            for r in self.lines
            if r.get("kind") == "failure_auto_file_spool" and r.get("spool_id") not in self.consumed
        ]

    def mark_consumed(self, spool_id: str) -> None:
        self.consumed.add(spool_id)


class FakeForge:
    """Test double recording every client call the filer makes.

    search_issues mirrors the REAL repo-scoped list endpoint (fix #122
    gate): server q= filters by marker-in-TITLE only (like real Gitea q),
    NOT by mirroring the client's filter — the old fake filtered the same
    way the filer did, making the filer's own check untestable.
    """

    def __init__(self, search_results=None, fail_with: Exception | None = None):
        self.search_results = search_results if search_results is not None else []
        self.fail_with = fail_with
        self._next_number = 7
        self.created: list[dict] = []
        self.commented: list[dict] = []
        self.searched: list[str] = []

    def search_issues(self, query: str) -> list[dict]:
        self.searched.append(query)
        if self.fail_with is not None:
            raise self.fail_with
        return [i for i in self.search_results if query in str(i.get("title", ""))]

    def create_issue(self, title: str, body: str) -> dict:
        if self.fail_with is not None:
            raise self.fail_with
        self.created.append({"title": title, "body": body})
        issue = {"number": self._next_number, "title": title, "state": "open"}
        self._next_number += 1
        self.search_results.append(dict(issue))  # findable by later searches
        return issue

    def add_comment(self, issue_number: int, body: str) -> dict:
        if self.fail_with is not None:
            raise self.fail_with
        self.commented.append({"issue_number": issue_number, "body": body})
        return {"created": True}

    def get_issue_state(self, issue_number: int) -> dict:
        return {"state": "open"}

    def list_open_issues(self) -> list[dict]:
        return []


@pytest.fixture(autouse=True)
def _real_journal(tmp_path, monkeypatch):
    """Point the canonical journal at tmp so emit() never touches real data."""
    monkeypatch.setattr("rune.event_bus._store_path", lambda: tmp_path / "events.jsonl")
    return None


# ---------------------------------------------------------------- signature


def test_signature_is_subsystem_and_error_class():
    assert signature_of("memory_selector", "RuntimeError: boom") == ("memory_selector", "RuntimeError")


def test_signature_from_plain_error_string_takes_first_token():
    assert signature_of("cron", "Connection refused to forge") == ("cron", "Connection")


def test_signature_accepts_exception_instance():
    try:
        raise ValueError("bad payload")
    except ValueError as exc:
        assert signature_of("persona", exc) == ("persona", "ValueError")


def test_error_class_of_exception_and_string():
    assert error_class_of(RuntimeError("x")) == "RuntimeError"
    assert error_class_of("TimeoutError: read timed out") == "TimeoutError"
    assert error_class_of("connection reset by peer") == "connection"


def test_error_class_of_caps_single_token_at_64_chars():
    # #122 gate: a 10KB single-token error must not make a 10KB marker.
    huge = "E" * 10000
    capped = error_class_of(f"{huge}: boom")
    assert len(capped) == 64
    assert len(marker_for("subsystem", capped)) < 100  # marker stays bounded
    assert error_class_of("RuntimeError: ok") == "RuntimeError"  # short: untouched


def test_marker_format():
    assert marker_for("memory_selector", "RuntimeError") == MARKER


# ---------------------------------------------------------------- file path


def test_first_occurrence_creates_issue_with_marker_title():
    forge = FakeForge()
    filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
    filer.file("memory_selector", "RuntimeError: boom", tier=1)
    assert len(forge.created) == 1
    assert forge.created[0]["title"].startswith(MARKER)
    assert forge.commented == []


def test_repeat_occurrence_comments_on_existing_issue():
    forge = FakeForge(search_results=[{"number": 7, "title": MARKER + " memory_selector"}])
    filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
    filer.file("memory_selector", "RuntimeError: boom", tier=1)
    assert forge.created == []  # dedup: never a second issue per signature
    assert len(forge.commented) == 1
    assert "+1 occurrences" in forge.commented[0]["body"]
    assert "RuntimeError" in forge.commented[0]["body"]


def test_each_repeat_comments_never_creates():
    forge = FakeForge(search_results=[{"number": 7, "title": MARKER + " memory_selector"}])
    filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
    filer.file("memory_selector", "RuntimeError: boom", tier=1)
    filer.file("memory_selector", "RuntimeError: another payload", tier=1)
    assert forge.created == []
    assert len(forge.commented) == 2
    assert "another payload" in forge.commented[1]["body"]


def test_search_results_without_marker_in_title_are_ignored():
    # Server-side q= is fuzzy (title OR body); only a title match dedups.
    forge = FakeForge(search_results=[{"number": 9, "title": "unrelated discussion of the marker"}])
    filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
    filer.file("memory_selector", "RuntimeError: boom", tier=1)
    assert len(forge.created) == 1
    assert forge.commented == []


def test_tier_gate_t2_produces_zero_client_calls():
    forge = FakeForge()
    filer = FailureAutoFiler(forge, FakeJournal())
    filer.file("cron", "Connection refused", tier=2)
    assert forge.searched == []
    assert forge.created == []
    assert forge.commented == []


def test_tier_gate_t0_and_t1_file():
    forge = FakeForge()
    filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
    filer.file("memory_selector", "RuntimeError: a", tier=0)
    filer.file("warm_router", "RuntimeError: b", tier=1)
    assert len(forge.created) == 2


def test_create_issue_body_carries_tier_and_count():
    forge = FakeForge()
    filer = FailureAutoFiler(forge, FakeJournal())
    filer.file("memory_selector", "RuntimeError: boom", tier=1)
    body = forge.created[0]["body"]
    assert "tier=1" in body
    assert "occurrences=1" in body


def test_error_excerpt_is_truncated():
    forge = FakeForge()
    filer = FailureAutoFiler(forge, FakeJournal())
    filer.file("memory_selector", "RuntimeError: " + ("x" * 500), tier=1)
    body = forge.created[0]["body"]
    assert ("x" * 500) not in body
    assert ("x" * 100) in body


# ---------------------------------------------------------------- redaction


def test_redact_strips_query_string_credentials():
    assert (
        redact("GET https://forge.example.com/api?token=supersecret&page=2 failed")
        == "GET https://forge.example.com/api?token=<redacted>&page=2 failed"
    )
    assert "password=hunter2" not in redact("postgres://host/db?password=hunter2&x=1")
    assert "api_key=abcd" not in redact("https://api.example.com/v1?api_key=abcd&y=2")
    assert "key=zzz" not in redact("https://x.example/?key=zzz")


def test_redact_strips_url_basic_auth_and_dsn_credentials():
    out = redact("connect failed for postgres://alice:hunter2@db.example.com:5432/rune")
    assert "hunter2" not in out
    assert "alice" not in out
    assert "postgres://<redacted>:<redacted>@db.example.com:5432/rune" in out
    out2 = redact("redis://:s3cr3t@cache.example.com:6379/0")
    assert "s3cr3t" not in out2


def test_redact_strips_bearer_and_assignment_credentials():
    out = redact("Authorization: Bearer eyJhbGciOi.eyJzdWIi.SflKxwRJSMeK")
    assert "eyJhbGciOi" not in out
    assert "<redacted>" in out
    out2 = redact("login failed with password='hunter2' and secret=abcdef1234")
    assert "hunter2" not in out2
    assert "abcdef1234" not in out2


def test_redact_keeps_error_class_and_shape_visible():
    out = redact("RuntimeError: connection to https://user:pass@forge.example.com refused (port 443)")
    assert out.startswith("RuntimeError: ")
    assert "refused (port 443)" in out  # non-credential detail survives
    # Redaction is deterministic on the credential span only:
    body = "connection to https://user:pass@forge.example.com refused (port 443)"
    assert out == "RuntimeError: " + redact(body)
    assert redact("plain error, nothing sensitive") == "plain error, nothing sensitive"


def test_excerpt_redacts_credentials():
    # excerpt() is what lands in persistent forge issues and spool lines:
    # it must redact before returning.
    assert "hunter2" not in excerpt("postgres://bob:hunter2@db/rune down")
    assert "<redacted>" in excerpt("GET https://forge.example.com/api?token=topsecret")


def test_credentialed_error_never_lands_raw_in_issue_or_spool():
    forge = FakeForge(fail_with=ConnectionError("connect: https://user:hunter2@forge.example.com/api?token=t0psecret"))
    journal = FakeJournal()
    filer = FailureAutoFiler(forge, journal, batch_size=1)
    filer.file("memory_selector", "RuntimeError: dsn postgres://alice:s3cr3t@db.example.com/rune", tier=1)
    spooled = journal.unconsumed()[0]
    for blob in (json.dumps(forge.created), json.dumps(spooled)):
        for secret in ("hunter2", "s3cr3t", "t0psecret"):
            assert secret not in blob


# ---------------------------------------------------------------- spool path


def test_client_exception_spools_and_never_raises():
    forge = FakeForge(fail_with=ConnectionError("forge down"))
    journal = FakeJournal()
    filer = FailureAutoFiler(forge, journal)
    # Must NOT raise.
    filer.file("memory_selector", "RuntimeError: boom", tier=1)
    pending = journal.unconsumed()
    assert len(pending) == 1
    line = pending[0]
    assert line["kind"] == "failure_auto_file_spool"
    assert line["signature"] == ["memory_selector", "RuntimeError"]
    assert line["tier"] == 1
    assert "boom" in line["error"]
    assert "timestamp" in line
    assert forge.created == []


def test_spool_write_failure_still_never_raises():
    forge = FakeForge()
    journal = FakeJournal()

    def explode(record):
        raise OSError("disk full")

    journal.append = explode
    filer = FailureAutoFiler(forge, journal)
    # Neither forge failure nor journal failure may escape.
    filer.file("memory_selector", "RuntimeError: boom", tier=1)


def test_jsonl_spool_line_is_durable_and_json(tmp_path):
    path = tmp_path / "events.jsonl"
    forge = FakeForge(fail_with=ConnectionError("forge down"))
    spool = JsonlSpool(path)
    filer = FailureAutoFiler(forge, spool)
    filer.file("memory_selector", "RuntimeError: boom", tier=1)
    rows = [json.loads(line) for line in path.read_text().splitlines()]
    assert rows and rows[-1]["kind"] == "failure_auto_file_spool"
    assert spool.unconsumed()[0]["spool_id"] == rows[-1]["spool_id"]


# ---------------------------------------------------------------- protocol


def test_null_adapter_search_returns_empty():
    assert NullAdapter().search_issues("anything") == []


def test_filer_accepts_protocol_clients():
    # Structural check: the fake satisfies ForgeClient incl. search_issues.
    assert issubclass(FakeForge, ForgeClient)
    assert issubclass(NullAdapter, ForgeClient)


# ------------------------------------------------- queue / worker / kill tests


def test_submit_enqueues_and_drain_delivers_writes():
    forge = FakeForge(search_results=[{"number": 7, "title": MARKER + " memory_selector"}])
    filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
    filer.submit("cron", "Connection refused", tier=2)  # gated: zero client calls
    filer.submit("memory_selector", "RuntimeError: a", tier=1)
    filer.submit("memory_selector", "RuntimeError: b", tier=1)
    filer.drain(timeout=5)
    assert forge.created == []  # existing issue deduped
    # Worker may batch both queued occurrences into one comment, or handle
    # them separately — either way: comments only, never a second issue.
    assert 1 <= len(forge.commented) <= 2


def test_queue_is_bounded_and_overflow_drops_spools_and_logs(caplog):
    # #122 gate: the queue must be bounded (a forge outage must not balloon
    # memory) and an overflow occurrence is DROPPED from the queue but
    # SPOOLED to the journal (nothing lost) + logged — never blocked.
    import logging

    forge = FakeForge()
    journal = FakeJournal()
    filer = FailureAutoFiler(forge, journal, wait=True)
    assert filer._queue.maxsize == 1000
    for i in range(1000):
        filer.submit("memory_selector", f"RuntimeError: fill {i}", tier=1)
    assert filer._queue.full()
    with caplog.at_level(logging.WARNING, logger="rune.failure_auto_filer"):
        filer.submit("memory_selector", "RuntimeError: dropped-occurrence", tier=1)
    assert filer._queue.full()  # queue did not grow past maxsize
    pending = journal.unconsumed()
    assert any(r["error"].endswith("dropped-occurrence") for r in pending)  # spooled
    assert any("queue full" in rec.getMessage() for rec in caplog.records)  # logged


def test_worker_exception_never_propagates():
    forge = FakeForge(fail_with=ConnectionError("forge down"))
    journal = FakeJournal()
    filer = FailureAutoFiler(forge, journal)
    filer.submit("memory_selector", "RuntimeError: boom", tier=1)
    filer.drain(timeout=5)  # worker hit the exception; must not raise here
    assert len(journal.unconsumed()) == 1


def test_t2_kill_test_zero_client_calls_through_worker():
    # Frozen kill test: a T2 failure must produce ZERO client calls even
    # when the worker thread drains the queue.
    forge = FakeForge()
    filer = FailureAutoFiler(forge, FakeJournal())
    filer.submit("cron", "Connection refused", tier=2)
    filer.submit("cron", "JSONDecodeError while parsing", tier=2)
    filer.drain(timeout=5)
    assert forge.searched == []
    assert forge.created == []
    assert forge.commented == []


def test_forge_down_kill_test_never_raises_into_failure_path():
    forge = FakeForge(fail_with=ConnectionError("forge down"))
    journal = FakeJournal()
    filer = FailureAutoFiler(forge, journal)
    try:
        filer.submit("memory_selector", "RuntimeError: boom", tier=1)
        filer.drain(timeout=5)
    except Exception as exc:  # pragma: no cover - the assertion below bites
        pytest.fail(f"filer raised into the failure path: {exc!r}")
    assert len(journal.unconsumed()) == 1


def test_event_bus_emit_unchanged_after_filer_import():
    # Spool rides the canonical journal; regular emit keeps working.
    event_bus.emit("user.message", source="t", payload={"k": "v"})