#!/usr/bin/env python3 """ci_event_wait.py — T16 §16C/§16D: event-driven CI qualification wait. Replaces the deploy.yml sleep-30 → query → repeat polling loop with the EXISTING W/R event bridge (src/rune/event_wait.py — the SAME primitive the builder adapter uses; §16E: one bridge, adapters differ only in identity and authority probe). Flow (mandate §16C): push / CI workflow starts ↓ register correlation (provider="gitea", event_type="ci_completed", run_ref="ci-", sha=) ↓ WAITING_FOR_EVENT (await_resume_event: select() on FIFO, no floor) ↓ ci.yml final job POSTs /process-event (the WAKE) ↓ daemon matches → resume_cmd fires → FIFO token → controller resumes ↓ canonical Gitea state INDEPENDENTLY re-read (deploy_qualify.py verdict) ↓ verdict The event is a WAKE, never truth. The final verdict always comes from deploy_qualify.py (required-checks context + exact-SHA ci.yml run, both legs). Watchdog deadline is a FAILURE DETECTOR, never the scheduler: - event arrives → authority re-read; pending → bounded re-arm until the overall deadline (ack-window semantics, same as the daemon's wake sweep). - event lost/suppressed → watchdog expires → ONE-SHOT authority query: terminal verdict → recovered truthfully (no invented wait); CI still running (non-terminal Gitea runs for the SHA) → re-arm; nothing alive and no verdict → truthful failure (deployment refused). Exit codes: 0 = qualified (success), 1 = refused (failure/timeout). Output: one JSON line with the §16D latency-proof timestamps. """ from __future__ import annotations import json import os import subprocess import sys import time from pathlib import Path _SCRIPT_DIR = Path(__file__).resolve().parent sys.path.insert(0, str(_SCRIPT_DIR)) sys.path.insert(0, str(_SCRIPT_DIR.parent / "src")) sys.path.insert(0, str(_SCRIPT_DIR)) from rune import event_wait as ew # noqa: E402 import completion_file as cf # noqa: E402 REPO_ROOT = _SCRIPT_DIR.parent QUALIFY_SCRIPT = _SCRIPT_DIR / "deploy_qualify.py" def authority_verdict(sha: str = "") -> str: """ONE-SHOT canonical Gitea verdict via deploy_qualify.py — the only completion truth. Never parses the event payload. H-PATCH H4: the target sha is passed EXPLICITLY (as the subprocess's --sha argument) — ambient DEPLOY_SHA can no longer change which operation is verified. The env var remains only as a CLI backward compat for deploy_qualify.py invoked standalone.""" cmd = [sys.executable, str(QUALIFY_SCRIPT)] if sha: cmd += ["--sha", sha] proc = subprocess.run( cmd, capture_output=True, text=True, timeout=120, env={k: v for k, v in os.environ.items() if k != "DEPLOY_SHA"} if sha else dict(os.environ, PYTHONPATH=""), ) out = (proc.stdout or "").strip().splitlines() return (out[-1] if out else "pending").strip() # ── T21.5 §4: operation-specific completion authority ────────────────────── # # T21.4 root cause: the wait loop had ONE hard-coded authority # (deploy qualification = master-ref CI) while operations waited on # branch CI, PR checks, or merges. Wrong authority => nonterminal # returns => the MODEL had to re-arm (the generator of the sleeps). # The wait verifier MUST match the operation being waited for. OPERATION_SPECS = { # op_type: { # workflow_path: Actions run path filter (None = any), # head_branches: head-branch filter (None = any), # authority: which canonical verifier decides completion, # event_types: expected wake event types, # terminal: terminal verdicts, failure: failure verdicts, # } "branch_ci": { "workflow_path": "ci.yml", "head_branches": None, "authority": "ci_runs", # any completed run for this SHA "event_types": ("ci_completed",), "terminal": ("success",), "failure": ("failure", "cancelled"), }, "pull_request_checks": { "workflow_path": None, # all PR-triggered runs for SHA "head_branches": None, "authority": "ci_runs", "event_types": ("ci_completed",), "terminal": ("success",), "failure": ("failure", "cancelled"), }, "master_ci": { "workflow_path": "ci.yml", "head_branches": ("master",), "authority": "deploy_qualify", # full two-leg qualification "event_types": ("ci_completed",), "terminal": ("success",), "failure": ("failure", "cancelled"), }, "deploy": { "workflow_path": "deploy.yml", "head_branches": ("master",), "authority": "deploy_qualify", "event_types": ("ci_completed", "deploy_completed"), "terminal": ("success",), "failure": ("failure", "cancelled"), }, "release": { "workflow_path": None, "head_branches": ("master",), "authority": "deploy_qualify", "event_types": ("ci_completed", "release_completed"), "terminal": ("success",), "failure": ("failure", "cancelled"), }, } DEFAULT_OP_TYPE = "master_ci" def normalize_verdict(verdict: str) -> str: """T21.5 §18 audit fix: pending-class verdicts are ALL nonterminal. deploy_qualify emits "pending (probe failed)" for a transient probe failure — that is pending, never a terminal answer. The wait loop must not return control to the model over a string variant.""" v = (verdict or "").strip().lower() if v.startswith("pending"): return "pending" return (verdict or "").strip() def op_spec(op_type: str) -> dict: spec = OPERATION_SPECS.get(op_type) if spec is None: raise ValueError( f"unknown op_type {op_type!r} — must be one of " f"{sorted(OPERATION_SPECS)} (the wait verifier must match " "the operation being waited for)") return spec def op_authority_verdict(op_type: str, sha: str) -> str: """Canonical completion verdict FOR THIS OPERATION TYPE. ci_runs — the op's own Actions runs for this exact SHA: every required workflow run completed; terminal iff every run of the op's workflow set concluded success (branch/PR ops are their own truth; deploy qualification is NOT their authority). deploy_qualify — the two-leg deploy qualification (master CI + required-checks) via deploy_qualify.py. """ spec = op_spec(op_type) authority = spec["authority"] if authority == "deploy_qualify": v = normalize_verdict(authority_verdict(sha)) if v != "pending": return v # E-PATCH resilience: Gitea's combined commit-status endpoint can # 404 for the newest commit (observed live on auto-version bump # commits — the status object materializes late/never). The runs # ARE the same evidence the two legs read; fall back to the # operation-scoped runs verdict so authority stays resolvable. if _status_endpoint_unreadable(sha): return _ci_runs_verdict(spec, sha) return v if authority == "ci_runs": return _ci_runs_verdict(spec, sha) raise ValueError(f"unknown authority {authority!r} for {op_type!r}") def _status_endpoint_unreadable(sha: str) -> bool: """True when the combined commit-status endpoint 404s for the SHA (index lag / never-materialized status object) — the documented fallback trigger, verified against the API once per call.""" try: sys.path.insert(0, str(_SCRIPT_DIR)) import gitea_auth # noqa: E402 st, _body = gitea_auth.api(f"/commits/{sha}/status") return st == 404 except Exception: # noqa: BLE001 return False def _ci_runs_verdict(spec: dict, sha: str) -> str: """Operation-scoped Actions-run verdict (import the API client from deploy_qualify to keep ONE Gitea client).""" sys.path.insert(0, str(_SCRIPT_DIR)) import deploy_qualify as dq # noqa: E402 import gitea_auth # noqa: E402 server = os.environ.get("GITEA_SERVER", getattr(gitea_auth, "DEFAULT_SERVER", "")).rstrip("/") repo = os.environ.get("GITEA_REPO", getattr(gitea_auth, "DEFAULT_REPO", "")) sha_env = sha # H-PATCH: explicit parameter is the only authority token = gitea_auth.get_token() if gitea_auth.has_token() else "" if not (server and repo and sha_env and token): return "pending" # cannot verify -> not verified (fail closed) runs = dq._fetch_runs(server, repo, sha_env, token) if runs is None: return "pending" # probe failure -> keep waiting wfp = spec["workflow_path"] if wfp is not None: runs = [r for r in runs if str(r.get("path", "")).startswith(wfp + "@")] branches = spec["head_branches"] if branches is not None: runs = [r for r in runs if str(r.get("head_branch", "")) in branches] if not runs: return "pending" # no runs visible yet -> not terminal for r in runs: if r.get("status") != "completed": return "pending" # still running if r.get("conclusion") not in ("success", "skipped"): return "failure" # any non-success conclusion return "success" def op_ci_alive(op_type: str, sha: str) -> bool: """Liveness probe FOR THIS OPERATION TYPE: any non-terminal run in the op's run set (failure DETECTOR, never the scheduler).""" sys.path.insert(0, str(_SCRIPT_DIR)) import deploy_qualify as dq # noqa: E402 import gitea_auth # noqa: E402 spec = op_spec(op_type) server = os.environ.get("GITEA_SERVER", getattr(gitea_auth, "DEFAULT_SERVER", "")).rstrip("/") repo = os.environ.get("GITEA_REPO", getattr(gitea_auth, "DEFAULT_REPO", "")) sha_env = sha # H-PATCH: explicit parameter is the only authority token = gitea_auth.get_token() if gitea_auth.has_token() else "" if not (server and repo and sha_env and token): return False runs = dq._fetch_runs(server, repo, sha_env, token) if not runs: return False wfp = spec["workflow_path"] if wfp is not None: runs = [r for r in runs if str(r.get("path", "")).startswith(wfp + "@")] branches = spec["head_branches"] if branches is not None: runs = [r for r in runs if str(r.get("head_branch", "")) in branches] terminal = {"success", "failure", "cancelled", "skipped"} return any(str(r.get("status", "")).lower() not in terminal for r in runs) def authority_after_wake(op_type: str = DEFAULT_OP_TYPE, sha: str = "", max_tries: int = 3, gap_secs: int = 4) -> str: """(sha is EXPLICIT — ambient env never selects the target.)""" """Post-wake authority read. The daemon's own wake sweep uses a bounded event-triggered retry (4s ×3) for the job-finished-microseconds- ago ack window; a verdict read here can hit the same window. This is a bounded schedule, not a polling loop — 3 reads max, then back to the event wait / watchdog.""" verdict = "pending" for attempt in range(max_tries): verdict = op_authority_verdict(op_type, sha) if verdict != "pending": return verdict if attempt < max_tries - 1: time.sleep(gap_secs) return verdict def ci_alive() -> bool: """LEGACY liveness probe (master-ref deploy runs) — kept for callers predating op specs; run() uses op_ci_alive(op_type, sha).""" return op_ci_alive(DEFAULT_OP_TYPE, os.environ.get("DEPLOY_SHA", "")) def _set_cognition_state(op: str, state: str) -> None: """E-PATCH §12: stamp cognitive quiescence on the durable record so a fresh process knows cognition is parked (WAITING_EXTERNAL) vs resumable (RESUMABLE). Best-effort: observability never breaks the wait.""" try: rec = ew.operation_wait(op) if rec is None: return wait_file = ew.wr_root() / "waits" / f"{rec.get('wait_id')}.json" if not wait_file.is_file(): return data = json.loads(wait_file.read_text()) data.setdefault("ext", {})["continuation_state"] = state wait_file.write_text(json.dumps(data, indent=1, sort_keys=True)) except (OSError, ValueError): pass def run(*, sha: str, deadline_secs: int, watchdog_secs: int = 300, op_type: str = DEFAULT_OP_TYPE, task_id: str = "", daemon_url: str = ew.DEFAULT_DAEMON_URL) -> int: """T21.5 §3: ONE LOGICAL AWAIT. One logical external wait is one invocation of this function. All nonterminal conditions (event-seen-authority-pending, watchdog expiry with live runs, bounded authority-retry exhausted) re-arm INTERNALLY — control returns to the caller ONLY on a terminal verdict or on real deadline expiry. Watchdogs are failure detectors inside this abstraction, never the scheduler and never completion truth. """ spec = op_spec(op_type) # unknown op_type refused upfront op = f"ci-{sha}" # A-PATCH: the authority subprocess reads DEPLOY_SHA from the env; # python-level run() callers (tool normalization flow) must export it. if sha and not os.environ.get("DEPLOY_SHA"): os.environ["DEPLOY_SHA"] = sha # H-PATCH: resolve short SHAs to full 40-char — _fetch_runs and # deploy_qualify match on the FULL sha; a 12-char prefix would # silently fail equality and park the waiter until deadline # (the 962e351 root cause). if sha and len(sha) < 40: try: full = subprocess.run( ["git", "-C", str(_SCRIPT_DIR.parent), "rev-parse", sha], capture_output=True, text=True, timeout=10) if full.returncode == 0: sha = full.stdout.strip() os.environ["DEPLOY_SHA"] = sha except (OSError, subprocess.TimeoutExpired): pass t0 = time.time() # REAL_ASYNC_BRIDGE: FIFO is replaced by completion-file inotify. # The completion file write is the wake signal; inotify is the # kernel-level zero-latency, zero-CPU wake primitive. # Event replay (T16 §16C live finding): on a serialized runner the # completion event usually arrives BEFORE the waiter registers. # A matching durable event means the wake already happened — skip # the rendezvous entirely and go straight to the authority read. prior = ew.find_matching_event("gitea", "ci_completed", op) if prior is not None and prior.get("exit_code") is not None: rec = ew.register_process_wait( op, project="rune", objective=f"event-driven {op_type} qualification for {sha}", watched_for="Gitea CI completion event (replayed from the " "durable event log)", provider="gitea", event_type="ci_completed", sha=sha, watchdog_secs=min(watchdog_secs, deadline_secs), continuation_state="RESUMABLE", # event already arrived op_type=op_type, authority=spec["authority"]) registered_at = rec["created_at"] event_at = time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime()) resumed_at = event_at # the event is a WAKE, never truth — canonical authority decides verdict = normalize_verdict(authority_after_wake(op_type, sha)) if verdict == "pending" and not op_ci_alive(op_type, sha): verdict = normalize_verdict(op_authority_verdict(op_type, sha)) mode = ("event_replayed" if verdict != "pending" else "authority_unresolved") verdict_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) out = { "sha": sha, "op_type": op_type, "authority": spec["authority"], "registered_at": registered_at, "event_received_at": event_at, "controller_resumed_at": resumed_at, "verdict_at": verdict_at, "mode": mode, "rearm_reason": "completion event replayed from durable log " "(arrived before registration)", "verdict": verdict, "wall_secs": round(time.time() - t0, 3), } print(json.dumps(out)) return 0 if verdict == "success" else 1 rec = ew.register_process_wait( op, project="rune", objective=f"event-driven {op_type} qualification for {sha}", watched_for="Gitea CI completion event (wake; verdict re-read " f"from canonical {spec['authority']} authority)", provider="gitea", event_type="ci_completed", sha=sha, watchdog_secs=min(watchdog_secs, deadline_secs), continuation_state="WAITING_EXTERNAL", op_type=op_type, authority=spec["authority"], task_id=task_id, # §9: bind the operation to the calling task resume_mode="self") # §3: waiter owns its continuation via # completion-file/inotify; the daemon has NO # session-delivery role for this wait registered_at = rec["created_at"] _set_cognition_state(op, "WAITING_EXTERNAL") # cognition parked deadline = time.monotonic() + deadline_secs verdict = "pending" mode = "event" event_at = "" resumed_at = "" rearm_reason = "" rearm_count = 0 while time.monotonic() < deadline: remaining = max(1, int(deadline - time.monotonic())) # REAL_ASYNC_BRIDGE: block on inotify for the completion file. # Zero CPU, zero arbitrary timer. The inotify timeout is a # FAILURE DETECTOR — if no event arrives within the watchdog # window, we check the authority directly (lost-event recovery). # The deadline is containment only. watch_timeout = min(remaining, watchdog_secs) result = cf.wait_for_completion( op, timeout_secs=watch_timeout) # REAL_ASYNC_BRIDGE: check durable event log AND wait record # status on EVERY cycle — the daemon may have matched and # transitioned the record via /process-event. wr = ew.operation_wait(op) if wr and wr.get("status") in ("COMPLETED", "FAILED", "READY_TO_RESUME"): event_at = event_at or time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime()) resumed_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) verdict = normalize_verdict( authority_after_wake(op_type, sha)) if verdict != "pending": mode = "event" break prior = ew.find_matching_event("gitea", "ci_completed", op) if prior is not None and prior.get("exit_code") is not None: event_at = event_at or time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime()) resumed_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) verdict = normalize_verdict( authority_after_wake(op_type, sha)) if verdict != "pending": mode = "event" break if result is not None: # completion file exists: event arrived event_at = event_at or time.strftime( "%Y-%m-%dT%H:%M:%SZ", time.gmtime()) resumed_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) verdict = normalize_verdict( authority_after_wake(op_type, sha)) if verdict != "pending": mode = "event" break rearm_count += 1 rearm_reason = ("completion file seen; authority " "pending — re-subscribing") continue # inotify timeout: no event within watchdog window # FAILURE DETECTOR: check authority directly verdict = normalize_verdict( op_authority_verdict(op_type, sha)) if verdict != "pending": mode = "watchdog_authority_recovered" break rearm_count += 1 rearm_reason = ("watchdog expiry; op pending (not started " "or lagging) — re-subscribing inotify") continue # REAL_ASYNC_BRIDGE: no FIFO to close. The completion file is durable # and survives restarts. Lease release is not needed (no lease system # for file-based delivery). if verdict == "pending": if mode == "event": mode = "deadline_pending" verdict = normalize_verdict(op_authority_verdict(op_type, sha)) # final one-shot truth _set_cognition_state(op, "RESUMABLE" if verdict != "pending" else "RECOVERY_REQUIRED") verdict_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) # §2 STATE OWNERSHIP: the WAITER owns the OPERATION domain. On a # terminal authority verdict, stamp the durable record terminal with # the evidence — the daemon's session-delivery machinery must never # (and can no longer) overwrite this truth. Also updates the parked # continuation_state so fresh-process recovery sees reality. try: rec2 = ew.operation_wait(op) or rec if verdict != "pending": rec2["status"] = "COMPLETED" if verdict == "success" else "FAILED" rec2["verdict"] = verdict rec2["resume_evidence"] = { "kind": "authority_verified_terminal", "verdict": verdict, "mode": mode, "authority": spec["authority"], "sha": sha, "op_type": op_type, "verdict_at": verdict_at, } rec2["ext"] = {**rec2.get("ext", {}), "continuation_state": ("RESUMABLE" if verdict != "pending" else "RECOVERY_REQUIRED")} ew._load_waitstore().save(rec2) except Exception as _e: # noqa: BLE001 — record stamping must never corrupt the envelope (the envelope + authority remain the truth) print(f"ci_event_wait: terminal record stamp failed: {_e}", file=sys.stderr) out = { "sha": sha, "op_type": op_type, "authority": spec["authority"], "registered_at": registered_at, "event_received_at": event_at, "controller_resumed_at": resumed_at, "verdict_at": verdict_at, "mode": mode, "rearm_reason": rearm_reason, "internal_rearm_count": rearm_count, "verdict": verdict, "wall_secs": round(time.time() - t0, 3), } print(json.dumps(out)) return 0 if verdict == "success" else 1 KNOWN_FLAGS = ("--sha", "--deadline", "--op-type", "--watchdog", "--task-id") def main() -> int: """§5 CLI CONTRACT: no silent argument loss. Every flag is either explicitly supported (and takes effect) or loudly rejected — an acceptance envelope must reflect the configuration actually used.""" sha = os.environ.get("DEPLOY_SHA", "") args = sys.argv[1:] deadline = 900 op_type = DEFAULT_OP_TYPE watchdog = 300 task_id = "" i = 0 while i < len(args): a = args[i] if a == "--sha": sha = args[i + 1]; i += 2 elif a == "--deadline": deadline = int(args[i + 1]); i += 2 elif a == "--op-type": op_type = args[i + 1]; i += 2 elif a == "--watchdog": watchdog = int(args[i + 1]); i += 2 elif a == "--task-id": task_id = args[i + 1]; i += 2 elif a.startswith("--"): print(json.dumps({"error": f"unknown flag {a!r}", "known_flags": list(KNOWN_FLAGS), "contract": "no silent argument loss"})) return 2 else: print(json.dumps({"error": f"unexpected positional {a!r}", "known_flags": list(KNOWN_FLAGS)})) return 2 if not sha: print(json.dumps({"error": "no sha (DEPLOY_SHA or --sha required)"})) return 1 # A-PATCH fix: run()'s authority subprocess (deploy_qualify.py) reads # DEPLOY_SHA from the environment — a CLI `--sha` consumer previously # got a silent "pending" verdict forever because the var was never # exported. Export it so the canonical authority can resolve. os.environ["DEPLOY_SHA"] = sha return run(sha=sha, deadline_secs=deadline, op_type=op_type, watchdog_secs=watchdog, task_id=task_id) if __name__ == "__main__": sys.exit(main())