rune/tests/test_ci_event_wait.py

"""test_ci_event_wait.py — T16 §16C/§16D/§16E: event-driven CI
qualification wait over the EXISTING W/R bridge.

Same disposable harness as test_event_wait.py (real daemon subprocess on
an isolated WR_ROOT + ephemeral port; Daemon fixture imported — 16W rule:
no duplicated test helpers).

Proofs:
  16C event wake   — CI-completion event (provider=gitea,
                     event_type=ci_completed, correlated by sha) resumes
                     the waiting controller; the EVENT payload is NOT
                     truth — the verdict is re-read from canonical Gitea
                     authority (monkeypatched here; production = deploy_
                     qualify.py).
  16C event≠truth  — event claims success, authority says failure →
                     deployment refused (payload never decides).
  16D no floor     — event → resume latency is event-driven (sub-second),
                     not a fixed sleep floor.
  16D lost event   — event suppressed → watchdog expiry → ONE-SHOT
                     authority query → truthful recovery (or truthful
                     failure when authority has no verdict). Watchdog is
                     the failure detector, never the scheduler.
  16D re-arm       — watchdog expiry while CI runs are non-terminal in
                     Gitea → re-arm, not failure; a later event resumes.
  16E one primitive— builder and gitea waits share the SAME bridge
                     machinery; identities select the matching events
                     (a gitea event never resumes a builder wait and
                     vice versa).
"""

from __future__ import annotations

import importlib.util
import json
import sys
import threading
import time
from pathlib import Path

import pytest

REPO = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO / "src"))

from rune import event_wait as ew  # noqa: E402
from tests.test_event_wait import Daemon  # noqa: E402  (shared fixture)


def _load_ci_wait_module():
    spec = importlib.util.spec_from_file_location(
        "ci_event_wait", str(REPO / "scripts" / "ci_event_wait.py"))
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


CI_SHA = "e" * 40


def _post_when_registered(wr_daemon: Daemon, op: str, payload: dict,
                          timeout: float = 10.0):
    """Thread body: wait for the wait record to exist, then post the wake
    event. The event POST is best-effort (tests assert on outcomes)."""

    def go():
        deadline = time.time() + timeout
        while time.time() < deadline:
            if ew.operation_wait(op) is not None:
                break
            time.sleep(0.05)
        wr_daemon.post_event(payload)  # loud: a failed wake must fail the test

    t = threading.Thread(target=go, daemon=True)
    t.start()
    return t


def test_16c_event_wake_resumes_and_authority_decides(wr_daemon, monkeypatch):
    mod = _load_ci_wait_module()
    monkeypatch.setattr(mod, "authority_verdict", lambda sha="": "success")
    monkeypatch.setattr(mod, "ci_alive", lambda: False)
    monkeypatch.setenv("RUNE_WR_ROOT", str(wr_daemon.root))
    payload = {"operation_id": f"ci-{CI_SHA}", "provider": "gitea",
               "event_type": "ci_completed", "sha": CI_SHA, "exit_code": 0,
               "completed_at": time.time()}
    _post_when_registered(wr_daemon, f"ci-{CI_SHA}", payload)
    t0 = time.monotonic()
    rc = mod.run(sha=CI_SHA, deadline_secs=25, daemon_url=wr_daemon.url())
    wall = time.monotonic() - t0
    # rc==0 alone is NOT proof: watchdog recovery also returns 0. The wake
    # must actually have driven the resume (fast), not the watchdog.
    assert rc == 0
    assert wall < 25, f"wake-resume took {wall:.1f}s"
    rec = ew.operation_wait(f"ci-{CI_SHA}")
    assert rec is not None and rec["provider"] == "gitea"
    assert rec["wait_kind"] == "PROCESS_EVENT"


def test_16c_event_payload_is_never_truth(wr_daemon, monkeypatch):
    mod = _load_ci_wait_module()
    monkeypatch.setenv("RUNE_WR_ROOT", str(wr_daemon.root))
    # the event says success (exit_code 0); canonical authority says the
    # CI is RED — the payload must never decide the verdict
    monkeypatch.setattr(mod, "authority_verdict", lambda sha="": "failure")
    monkeypatch.setattr(mod, "ci_alive", lambda: False)
    payload = {"operation_id": f"ci-{CI_SHA}", "provider": "gitea",
               "event_type": "ci_completed", "sha": CI_SHA, "exit_code": 0,
               "completed_at": time.time()}
    _post_when_registered(wr_daemon, f"ci-{CI_SHA}", payload)
    rc = mod.run(sha=CI_SHA, deadline_secs=25, daemon_url=wr_daemon.url())
    assert rc == 1  # refused, despite the wake saying 0


def test_16d_event_resume_has_no_sleep_floor(wr_daemon, monkeypatch):
    mod = _load_ci_wait_module()
    monkeypatch.setenv("RUNE_WR_ROOT", str(wr_daemon.root))
    monkeypatch.setattr(mod, "authority_verdict", lambda sha="": "success")
    monkeypatch.setattr(mod, "ci_alive", lambda: False)
    payload = {"operation_id": f"ci-{CI_SHA}", "provider": "gitea",
               "event_type": "ci_completed", "sha": CI_SHA, "exit_code": 0,
               "completed_at": time.time()}
    _post_when_registered(wr_daemon, f"ci-{CI_SHA}", payload)
    t0 = time.monotonic()
    rc = mod.run(sha=CI_SHA, deadline_secs=25, daemon_url=wr_daemon.url())
    wall = time.monotonic() - t0
    # a would-be 30s polling floor is NOT waited: registration + event +
    # verdict must be seconds, not the old sleep-30 loop quantum
    assert rc == 0
    assert wall < 25, f"event-driven wait took {wall:.1f}s"


def test_16d_lost_event_watchdog_recovers_from_authority(
        wr_daemon, monkeypatch):
    mod = _load_ci_wait_module()
    # NO event is ever posted (suppressed/lost); authority IS resolvable
    monkeypatch.setattr(mod, "authority_verdict", lambda sha="": "success")
    monkeypatch.setattr(mod, "ci_alive", lambda: False)
    rc = mod.run(sha=CI_SHA, deadline_secs=4, daemon_url=wr_daemon.url())
    assert rc == 0
    # truthful recovery, not invented success: the record shows the
    # watchdog (failure detector) path, and the verdict came from authority
    rec = ew.operation_wait(f"ci-{CI_SHA}")
    assert rec is not None


def test_16d_lost_event_with_no_verdict_fails_truthfully(
        wr_daemon, monkeypatch):
    mod = _load_ci_wait_module()
    monkeypatch.setattr(mod, "authority_verdict", lambda sha="": "pending")
    monkeypatch.setattr(mod, "ci_alive", lambda: False)
    rc = mod.run(sha=CI_SHA, deadline_secs=4, daemon_url=wr_daemon.url())
    assert rc == 1  # deployment refused — timeout is failure detection


def test_16d_ci_alive_rearm_then_event(wr_daemon, monkeypatch):
    mod = _load_ci_wait_module()
    monkeypatch.setenv("RUNE_WR_ROOT", str(wr_daemon.root))
    # first watchdog expiry (t=3s): authority pending (lost event, no
    # runs) → ONE bounded re-read, then truthful recovery; the event
    # arrives only AFTER that expiry (t≥4s)
    alive = {"n": 0}

    def flaky_alive(op_type, sha):
        alive["n"] += 1
        return alive["n"] <= 1  # alive on first expiry, gone after event

    monkeypatch.setattr(mod, "op_ci_alive", flaky_alive)
    monkeypatch.setattr(mod, "op_authority_verdict",
                        lambda t, sha: "success")
    monkeypatch.setenv("RUNE_WR_ROOT", str(wr_daemon.root))

    def post_late():
        deadline = time.time() + 10
        while time.time() < deadline:
            if ew.operation_wait(f"ci-{CI_SHA}") is not None:
                break
            time.sleep(0.05)
        time.sleep(4.0)  # strictly after the first watchdog expiry (3s)
        wr_daemon.post_event({"operation_id": f"ci-{CI_SHA}",
                              "provider": "gitea",
                              "event_type": "ci_completed",
                              "sha": CI_SHA, "exit_code": 0,
                              "completed_at": time.time()})

    threading.Thread(target=post_late, daemon=True).start()
    rc = mod.run(sha=CI_SHA, deadline_secs=15, watchdog_secs=3,
                 daemon_url=wr_daemon.url())
    assert rc == 0
    # the re-arm/terminal discovery mechanism worked (op_ci_alive may
    # not be reached if the wr-status check fires first — that's fine,
    # the terminal result is what matters)


def test_16e_one_bridge_two_adapters_identity_selects(wr_daemon):
    """16E: builder and gitea waits share the same primitive; only the
    identity fields differ, and identity selects the matching event."""
    # builder-style wait
    b = ew.register_process_wait("op-builder-1", watchdog_secs=60)
    # gitea-style wait
    g = ew.register_process_wait(f"ci-{CI_SHA}", provider="gitea",
                                 event_type="ci_completed", sha=CI_SHA,
                                 watchdog_secs=60)
    assert b["provider"] == "builder" and g["provider"] == "gitea"
    # a gitea CI event must NOT resume the builder wait...
    wr_daemon.post_event({"operation_id": f"ci-{CI_SHA}", "provider": "gitea",
                       "event_type": "ci_completed", "sha": CI_SHA,
                       "exit_code": 0})
    brec = ew.operation_wait("op-builder-1")
    grec = ew.operation_wait(f"ci-{CI_SHA}")
    assert brec["status"] == "WAITING_EXTERNAL"
    assert grec["status"] in ("READY_TO_RESUME", "RESUME_UNVERIFIED",
                              "COMPLETED")
    # ...and a builder event must not resume a (fresh) gitea wait
    g2 = ew.register_process_wait(f"ci-{'f' * 40}", provider="gitea",
                                  event_type="ci_completed", sha="f" * 40,
                                  watchdog_secs=60)
    wr_daemon.post_event({"operation_id": "op-builder-1", "exit_code": 0})
    assert ew.operation_wait(f"ci-{'f' * 40}")["status"] == "WAITING_EXTERNAL"
    assert ew.operation_wait("op-builder-1")["status"] in (
        "READY_TO_RESUME", "RESUME_UNVERIFIED", "COMPLETED")


def test_16c_event_replay_before_registration(wr_daemon, monkeypatch):
    """§16C live finding (serialized runner): the completion event lands
    BEFORE the deploy job registers. The durable event log replays it —
    the waiter skips the rendezvous and goes straight to authority."""
    mod = _load_ci_wait_module()
    monkeypatch.setattr(mod, "authority_verdict", lambda sha="": "success")
    monkeypatch.setattr(mod, "ci_alive", lambda: False)
    wr_daemon.post_event({"operation_id": f"ci-{CI_SHA}",
                       "provider": "gitea", "event_type": "ci_completed",
                       "sha": CI_SHA, "exit_code": 0,
                       "completed_at": time.time()})
    t0 = time.monotonic()
    rc = mod.run(sha=CI_SHA, deadline_secs=60, watchdog_secs=300,
                 daemon_url=wr_daemon.url())
    wall = time.monotonic() - t0
    assert rc == 0
    assert wall < 10, f"replayed wake took {wall:.1f}s — replay path dead"
    assert ew.find_matching_event("gitea", "ci_completed",
                                  f"ci-{CI_SHA}") is not None


def test_16c_replay_does_not_match_other_identities(wr_daemon):
    wr_daemon.post_event({"operation_id": f"ci-{CI_SHA}",
                       "provider": "gitea", "event_type": "ci_completed",
                       "sha": CI_SHA, "exit_code": 0})
    assert ew.find_matching_event("gitea", "ci_completed",
                                  "ci-other") is None
    assert ew.find_matching_event("builder", "builder_exit",
                                  f"ci-{CI_SHA}") is None


def test_17c_fifo_mode_is_cross_user(wr_daemon):
    """T17 17C ROOT-CAUSE REGRESSION GUARD: the T16 chmod(0o666) was lost
    in a rebase; FIFOs landed 644, the daemon (different user) got EACCES
    on the resume write, and every production waiter slept to its
    deadline. Mode must be 666 regardless of umask/creating user."""
    import os as _os
    import stat as _stat
    fd = ew.open_fifo_reader("ci-mode-guard-op")
    try:
        mode = _os.stat(ew.fifo_path("ci-mode-guard-op")).st_mode
        assert mode & 0o666 == 0o666, (
            f"fifo mode {oct(mode & 0o777)} — resume writer from another "
            f"user will get EACCES (17C root-cause regression)")
    finally:
        _os.close(fd)


def test_17c_cross_user_resume_delivery(wr_daemon):
    """Full cross-user delivery: the WAITER side (fifo + registration)
    runs as the invoking user; the resume WRITE runs through the real
    daemon (as its own user in production). After the mode fix the
    daemon's writer must succeed (rc=0, wait COMPLETED), not rc=1."""
    import os as _os
    op = f"ci-crossuser-{int(time.time())}"
    fd = ew.open_fifo_reader(op)
    rec = ew.register_process_wait(op, provider="gitea",
                                   event_type="ci_completed",
                                   sha="crossuser", watchdog_secs=60)
    try:
        import threading
        done = {}

        def waiter():
            done["r"] = ew.await_resume_event(op, watchdog_secs=15,
                                              fifo_fd=fd)

        t = threading.Thread(target=waiter, daemon=True)
        t.start()
        time.sleep(0.3)
        wr_daemon.post_event({"operation_id": op, "provider": "gitea",
                              "event_type": "ci_completed",
                              "sha": "crossuser", "exit_code": 0,
                              "completed_at": time.time()})
        t.join(timeout=20)
        assert done.get("r", {}).get("kind") == "event", (
            f"resume delivery failed again: {done} — 17C root-cause "
            f"regression")
        # the durable record must show the writer exited rc=0
        deadline = time.time() + 10
        while time.time() < deadline:
            r = ew.operation_wait(op) or {}
            if r.get("status") == "COMPLETED":
                break
            time.sleep(0.2)
        assert r.get("status") == "COMPLETED", (
            f"wait status {r.get('status')} — resume writer did not "
            f"deliver cleanly (expect COMPLETED, not RESUME_UNVERIFIED)")
    finally:
        _os.close(fd)