diff --git a/bridge/daemon/completion_file.py b/bridge/daemon/completion_file.py index 67896895..44e77f4b 100644 --- a/bridge/daemon/completion_file.py +++ b/bridge/daemon/completion_file.py @@ -29,6 +29,7 @@ WHY NOT POLLING: that fires exactly when the filesystem changes — the latency tracks the real completion event, not a configured interval. """ + from __future__ import annotations import json @@ -38,18 +39,19 @@ import struct import time from pathlib import Path + def _wr_bridge() -> Path: # WR_ROOT = daemon/infra convention; RUNE_WR_ROOT = repo/scripts # convention. Both honored (WR_ROOT wins) so the daemon, the waiter, # and any isolated test root always resolve the SAME directory. - root = (os.environ.get("WR_ROOT") - or os.environ.get("RUNE_WR_ROOT") - or "/mnt/workdrive/wr-bridge") + root = os.environ.get("WR_ROOT") or os.environ.get("RUNE_WR_ROOT") or "/mnt/workdrive/wr-bridge" return Path(root) + def _resumes_dir() -> Path: return _wr_bridge() / "resumes" + # inotify constants (Linux) IN_CREATE = 0x100 IN_CLOSE_WRITE = 0x8 @@ -115,16 +117,14 @@ def _inotify_read(fd: int, timeout_secs: float | None = None) -> list[str]: # struct: wd(i32), mask(u32), cookie(u32), name_len(u32) wd, mask, cookie, name_len = struct.unpack_from("iIII", buf, pos) pos += 16 - name = buf[pos:pos + name_len].rstrip(b"\x00").decode("utf-8", "replace") + name = buf[pos : pos + name_len].rstrip(b"\x00").decode("utf-8", "replace") pos += name_len if mask & (IN_CREATE | IN_MOVED_TO): events.append(name) return events -def wait_for_completion(operation_id: str, *, - timeout_secs: float | None = None, - poll_callback=None) -> dict | None: +def wait_for_completion(operation_id: str, *, timeout_secs: float | None = None, poll_callback=None) -> dict | None: """Block until a completion file exists for this operation. Uses inotify for zero-latency, zero-CPU wake. Falls back to @@ -161,6 +161,7 @@ def wait_for_completion(operation_id: str, *, # 3. block in kernel on inotify — zero CPU, exact latency import select as _select + poll = _select.poll() poll.register(fd, _select.POLLIN) deadline = (time.monotonic() + timeout_secs) if timeout_secs else None diff --git a/bridge/daemon/daemon.py b/bridge/daemon/daemon.py index f4db1361..0437d8a3 100755 --- a/bridge/daemon/daemon.py +++ b/bridge/daemon/daemon.py @@ -32,6 +32,7 @@ The resume command is configurable (WR_RESUME_CMD, JSON list with {session_id}/{msg} placeholders) so the test harness can use a stub; the default is the real mechanism: `mimo run -s -m `. """ + from __future__ import annotations import fcntl, json, os, signal, subprocess, sys, threading, time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -58,8 +59,7 @@ WATCH_SECS = int(os.environ.get("WR_WATCH_SECS", "600")) # progress is an orphaned resume flow (thread died between lease acquire # and release). Must exceed the longest legitimate in-flight watch # (WATCH_SECS); default 3x. Env-overridable for deterministic tests. -ORPHAN_GRACE_SECS = int(os.environ.get("WR_ORPHAN_GRACE_SECS", - str(3 * WATCH_SECS))) +ORPHAN_GRACE_SECS = int(os.environ.get("WR_ORPHAN_GRACE_SECS", str(3 * WATCH_SECS))) TERM_GRACE_SECS = int(os.environ.get("WR_TERM_GRACE_SECS", "10")) MAX_ATTEMPTS = int(os.environ.get("WR_MAX_ATTEMPTS", "3")) _WR_ROOT = Path(os.environ.get("WR_ROOT", "") or Path(__file__).resolve().parent) @@ -74,25 +74,23 @@ def log(msg: str) -> None: f.write(f"{time.strftime('%H:%M:%S')} {msg}\n") -def resume_cmd(session_id: str, msg: str, wait_id: str = "", - override: list | None = None) -> list[str]: +def resume_cmd(session_id: str, msg: str, wait_id: str = "", override: list | None = None) -> list[str]: """T15 §15D: per-wait override wins; placeholders {session_id}/{msg}/ {wait_id}. A synthetic controller session (Scaffolding stage driver) is not a MiMo session — the global default cannot resume it, which is exactly why fixed-sleep polling crept back into T12-T14.""" if override: - return [c.replace("{session_id}", session_id) - .replace("{msg}", msg) - .replace("{wait_id}", wait_id) for c in override] + return [ + c.replace("{session_id}", session_id).replace("{msg}", msg).replace("{wait_id}", wait_id) for c in override + ] tmpl = os.environ.get("WR_RESUME_CMD") cmd = json.loads(tmpl) if tmpl else DEFAULT_RESUME_CMD - return [c.replace("{session_id}", session_id) - .replace("{msg}", msg).replace("{wait_id}", wait_id) - for c in cmd] + return [c.replace("{session_id}", session_id).replace("{msg}", msg).replace("{wait_id}", wait_id) for c in cmd] # ── process-tree evidence & ownership ───────────────────────────────────── + def pgid_members(pgid: int) -> list[int]: out = [] for d in Path("/proc").iterdir(): @@ -142,8 +140,7 @@ def stray_worker_pids(session_id: str, exclude_pgids: set[int] = ()) -> list[int if not d.name.isdigit(): continue try: - parts = [p.decode(errors="replace") - for p in (d / "cmdline").read_bytes().split(b"\0") if p] + parts = [p.decode(errors="replace") for p in (d / "cmdline").read_bytes().split(b"\0") if p] except OSError: continue if not parts: @@ -164,6 +161,7 @@ def stray_worker_pids(session_id: str, exclude_pgids: set[int] = ()) -> list[int # ── verification → transition (unchanged authority model) ───────────────── + def _authoritative_verify(rec: dict) -> tuple[str, dict]: """Authoritative verification with a deterministic test seam. @@ -183,11 +181,10 @@ def _authoritative_verify(rec: dict) -> tuple[str, dict]: verdict = spec.get(key, spec.get("default", "pending")) return verdict, {"stub": True, "key": key} import verify + if rec.get("wait_kind") == "RELEASE_DEPLOY": - return verify.verify_release_deploy( - rec.get("sha", ""), rec.get("repo_root", ""), rec.get("deployed_link", "")) - return verify.verify_run(rec.get("run_ref", ""), rec.get("sha", ""), - rec.get("expected", "success")) + return verify.verify_release_deploy(rec.get("sha", ""), rec.get("repo_root", ""), rec.get("deployed_link", "")) + return verify.verify_run(rec.get("run_ref", ""), rec.get("sha", ""), rec.get("expected", "success")) def try_transition(rec: dict, event: dict | None = None) -> dict: @@ -207,11 +204,8 @@ def try_transition(rec: dict, event: dict | None = None) -> dict: if exit_code is None: return rec # malformed event: stay WAITING (15E hostile case) verdict = "success" if exit_code == 0 else "failure" - evidence = {"process_event": True, - "exit_code": exit_code, - "operation_id": event.get("run_ref", "")} - log(f"verify {rec['wait_id']}: {verdict} " - f"{json.dumps(evidence)[:120]}") + evidence = {"process_event": True, "exit_code": exit_code, "operation_id": event.get("run_ref", "")} + log(f"verify {rec['wait_id']}: {verdict} {json.dumps(evidence)[:120]}") rec["status"] = "READY_TO_RESUME" rec["verdict"] = verdict if verdict == "failure": @@ -237,24 +231,29 @@ def try_transition(rec: dict, event: dict | None = None) -> dict: def continuation_msg(rec: dict) -> str: c = rec["continuation"] if rec.get("verdict") == "success": - return (f"External wait satisfied (wait {rec['wait_id']}).\n" - f"Original objective: {c.get('objective','')}\n" - f"Waited for: {c.get('waited_for','')}\n" - f"Authoritative result: workflow success verified via Gitea API.\n" - f"SHA/run: {rec.get('sha','')}/{rec.get('run_ref','')}\n" - f"Verify: {c.get('verify_when_woken','')}\n" - f"Continue from: {c.get('on_success','')}") + return ( + f"External wait satisfied (wait {rec['wait_id']}).\n" + f"Original objective: {c.get('objective', '')}\n" + f"Waited for: {c.get('waited_for', '')}\n" + f"Authoritative result: workflow success verified via Gitea API.\n" + f"SHA/run: {rec.get('sha', '')}/{rec.get('run_ref', '')}\n" + f"Verify: {c.get('verify_when_woken', '')}\n" + f"Continue from: {c.get('on_success', '')}" + ) ev = json.dumps(rec.get("failure_evidence", {}))[:400] - return (f"External wait finished with FAILURE (wait {rec['wait_id']}).\n" - f"Original objective: {c.get('objective','')}\n" - f"Authoritative result: workflow FAILURE verified via Gitea API.\n" - f"Failed job evidence: {ev}\n" - f"SHA/run: {rec.get('sha','')}/{rec.get('run_ref','')}\n" - f"Do NOT treat this as success. Continue from: {c.get('on_failure','')}") + return ( + f"External wait finished with FAILURE (wait {rec['wait_id']}).\n" + f"Original objective: {c.get('objective', '')}\n" + f"Authoritative result: workflow FAILURE verified via Gitea API.\n" + f"Failed job evidence: {ev}\n" + f"SHA/run: {rec.get('sha', '')}/{rec.get('run_ref', '')}\n" + f"Do NOT treat this as success. Continue from: {c.get('on_failure', '')}" + ) # ── resume driver (single-flight, tree-owned) ───────────────────────────── + def resume(rec: dict) -> None: """Run one continuation to completion. Runs on a dedicated worker thread. Preconditions: caller verified no inflight worker for this session.""" @@ -268,9 +267,11 @@ def resume(rec: dict) -> None: # stale relative to the waiter's terminal stamp. fresh = waitstore.load(wid) if fresh and fresh.get("status") in waitstore.TERMINAL: - log(f"resume {wid}: skipped — operation already terminal " + log( + f"resume {wid}: skipped — operation already terminal " f"({fresh['status']}); delivery worker never overrides " - f"operation truth") + f"operation truth" + ) return rec = fresh or rec msg = continuation_msg(rec) @@ -299,10 +300,13 @@ def resume(rec: dict) -> None: waitstore.save(rec) try: - proc = subprocess.Popen(resume_cmd(sid, msg, wid, - rec.get("resume_cmd")), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, start_new_session=True) + proc = subprocess.Popen( + resume_cmd(sid, msg, wid, rec.get("resume_cmd")), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) except Exception as e: waitstore.release_lease(sid, wid) rec["status"] = "READY_TO_RETRY" # spawn failed pre-delivery → retryable @@ -312,8 +316,7 @@ def resume(rec: dict) -> None: return pgid = proc.pid # start_new_session ⇒ pgid == child pid - waitstore.update_lease(sid, wait_id=wid, pid=proc.pid, pgid=pgid, - pid_starttime=waitstore._pid_starttime(proc.pid)) + waitstore.update_lease(sid, wait_id=wid, pid=proc.pid, pgid=pgid, pid_starttime=waitstore._pid_starttime(proc.pid)) log(f"launched {wid}: pid={proc.pid} pgid={pgid} attempt={rec['resume_attempts']}") timed_out = False @@ -337,12 +340,12 @@ def resume(rec: dict) -> None: # D+E: timeout is NOT success. Tree terminated; retry truthfully. if rec["resume_attempts"] >= MAX_ATTEMPTS: rec["status"] = "RESUME_UNVERIFIED" - rec["resume_note"] = (f"watch timeout x{rec['resume_attempts']}; " - f"process group {pgid} terminated; attempts exhausted") + rec["resume_note"] = ( + f"watch timeout x{rec['resume_attempts']}; process group {pgid} terminated; attempts exhausted" + ) else: rec["status"] = "READY_TO_RETRY" - rec["resume_note"] = (f"watch timeout; process group {pgid} terminated " - f"(TERM→KILL, verified); will retry") + rec["resume_note"] = f"watch timeout; process group {pgid} terminated (TERM→KILL, verified); will retry" log(f"timeout {wid}: tree {pgid} terminated, rc={rc}") elif rc == 0: # COMPLETION TRUTH (constraint 4): rc==0 means the headless worker @@ -352,7 +355,8 @@ def resume(rec: dict) -> None: terminal = "COMPLETED" if rec.get("verdict") == "success" else "FAILED" rec["status"] = terminal rec["resume_evidence"] = { - "kind": "worker_exited_rc0", "rc": 0, + "kind": "worker_exited_rc0", + "rc": 0, "output_tail": (stdout or "")[-400:], "finished_at": waitstore.now(), } @@ -361,7 +365,8 @@ def resume(rec: dict) -> None: # Nonzero rc: delivery/completion unproven. Honest, not schedulable. rec["status"] = "RESUME_UNVERIFIED" rec["resume_evidence"] = { - "kind": "worker_exited_nonzero", "rc": rc, + "kind": "worker_exited_nonzero", + "rc": rc, "output_tail": ((stdout or "") + (stderr or ""))[-400:], "finished_at": waitstore.now(), } @@ -372,6 +377,7 @@ def resume(rec: dict) -> None: # ── startup recovery (invariant F) ──────────────────────────────────────── + def startup_recovery() -> None: for rec in waitstore.all_waits(): if rec.get("status") != "RESUMING": @@ -388,8 +394,9 @@ def startup_recovery() -> None: survivors = terminate_tree(pgid) if pgid_members(pgid) else [] waitstore.release_lease(sid, wid) rec["status"] = "RESUME_UNVERIFIED" - rec["resume_note"] = ("adopted post-restart; outcome unverifiable" - + (f"; survivors killed: {survivors}" if survivors else "")) + rec["resume_note"] = "adopted post-restart; outcome unverifiable" + ( + f"; survivors killed: {survivors}" if survivors else "" + ) waitstore.save(rec) log(f"recover {wid}: adopted worker ended → RESUME_UNVERIFIED") else: @@ -419,17 +426,22 @@ def drive_ready() -> None: # target_kind="headless" (disposable/owned sessions) auto-dispatch. if rec.get("target_kind") != "headless": if not rec.get("hold_reason"): - rec["hold_reason"] = ("INTERACTIVE_AUTO_RESUME = NOT YET SAFE: " - "no supported MiMo busy/idle signal; " - "queued until operator releases") + rec["hold_reason"] = ( + "INTERACTIVE_AUTO_RESUME = NOT YET SAFE: " + "no supported MiMo busy/idle signal; " + "queued until operator releases" + ) waitstore.save(rec) continue - if not rec.get("resume_cmd"): + if rec.get("ext", {}).get("resume_mode") == "self": # REAL_ASYNC_BRIDGE §3: self-resuming wait (completion-file / # inotify waiter owns its continuation). The daemon has NO - # delivery role here — dispatching the legacy FIFO writer - # would stamp a phantom rc!=0 over the waiter's own truthful + # delivery role here — dispatching a delivery worker would + # stamp a phantom rc!=0 over the waiter's own truthful # terminal state (the canonical-state contradiction). + # NB: keyed on the explicit self marker, NOT on resume_cmd + # absence — records without inline resume_cmd may still be + # dispatchable via the WR_RESUME_CMD env template. continue with _inflight_lock: t = _inflight.get(sid) @@ -440,8 +452,7 @@ def drive_ready() -> None: if strays: log(f"drive: {rec['wait_id']} held, session busy {strays}") continue - th = threading.Thread(target=resume, args=(rec,), daemon=True, - name=f"resume-{rec['wait_id']}") + th = threading.Thread(target=resume, args=(rec,), daemon=True, name=f"resume-{rec['wait_id']}") _inflight[sid] = th # start INSIDE the lock: a just-created thread is not yet # is_alive() until start() runs — starting outside the lock let @@ -458,6 +469,7 @@ def reconcile_once() -> None: created = rec.get("created_at", "") try: import calendar + age = time.time() - calendar.timegm(time.strptime(created, "%Y-%m-%dT%H:%M:%SZ")) except ValueError: continue @@ -479,13 +491,13 @@ def orphan_sweep() -> int: (delivery unverifiable), never auto-relaunch. """ import calendar as _cal + healed = 0 for rec in waitstore.all_waits(): if rec.get("status") != "RESUMING": continue try: - updated = _cal.timegm(time.strptime( - rec.get("last_updated", ""), "%Y-%m-%dT%H:%M:%SZ")) + updated = _cal.timegm(time.strptime(rec.get("last_updated", ""), "%Y-%m-%dT%H:%M:%SZ")) except ValueError: continue if time.time() - updated < ORPHAN_GRACE_SECS: @@ -493,8 +505,7 @@ def orphan_sweep() -> int: wid, sid = rec["wait_id"], rec["session_id"] lease = waitstore.read_lease(sid) owner_dead = lease is not None and not waitstore._owner_alive(lease) - daemon_owned = (lease is not None - and lease.get("pid") == os.getpid()) + daemon_owned = lease is not None and lease.get("pid") == os.getpid() if lease is None or owner_dead or daemon_owned: if lease is not None: waitstore.release_lease(sid, wid) @@ -505,9 +516,9 @@ def orphan_sweep() -> int: rec["resume_note"] = ( "orphaned mid-run: resume flow died between lease acquire " "and delivery (" - + ("lease owner dead" if owner_dead else - "resume thread lost" if daemon_owned else - "no lease") + "); orphan sweep recovered") + + ("lease owner dead" if owner_dead else "resume thread lost" if daemon_owned else "no lease") + + "); orphan sweep recovered" + ) waitstore.save(rec) log(f"orphan-sweep {wid}: {rec['status']} — {rec['resume_note']}") healed += 1 @@ -543,6 +554,7 @@ def reconciler() -> None: # ── webhook receiver (wake signal only) ─────────────────────────────────── + def wake_sweep(event: dict, eid: str) -> dict: """Sweep WAITING_EXTERNAL waits through authoritative verification. @@ -565,8 +577,7 @@ def wake_sweep(event: dict, eid: str) -> dict: if rec["status"] != "WAITING_EXTERNAL": continue try: - age = time.time() - calendar.timegm( - time.strptime(rec.get("created_at", ""), "%Y-%m-%dT%H:%M:%SZ")) + age = time.time() - calendar.timegm(time.strptime(rec.get("created_at", ""), "%Y-%m-%dT%H:%M:%SZ")) except ValueError: continue if age < WAKE_MIN_AGE_SECS: @@ -581,8 +592,10 @@ def wake_sweep(event: dict, eid: str) -> dict: return verified, advanced, pending verified, advanced, pending = sweep() - log(f"local wake {eid}: verified={verified} advanced={advanced} " - f"pending={pending} sweep_ms={int((time.time() - arrival) * 1000)}") + log( + f"local wake {eid}: verified={verified} advanced={advanced} " + f"pending={pending} sweep_ms={int((time.time() - arrival) * 1000)}" + ) for attempt in range(1, WAKE_RETRIES + 1): # Retry while the sweep didn't conclusively advance anything: # pending verdicts (ack-window) AND fresh-wait skips (age gate). @@ -591,9 +604,11 @@ def wake_sweep(event: dict, eid: str) -> dict: break time.sleep(WAKE_RETRY_SECS) verified, advanced, pending = sweep() - log(f"local wake {eid}: retry{attempt} verified={verified} " + log( + f"local wake {eid}: retry{attempt} verified={verified} " f"advanced={advanced} pending={pending} " - f"total_ms={int((time.time() - arrival) * 1000)}") + f"total_ms={int((time.time() - arrival) * 1000)}" + ) return {"event_id": eid} @@ -614,7 +629,9 @@ class Handler(BaseHTTPRequestHandler): try: payload = json.loads(body) if body else {} except json.JSONDecodeError: - self.send_response(400); self.end_headers(); return + self.send_response(400) + self.end_headers() + return event = { "provider": str(payload.get("provider", "builder")), "event_type": str(payload.get("event_type", "builder_exit")), @@ -640,13 +657,14 @@ class Handler(BaseHTTPRequestHandler): # completion files (R10 idempotence by construction). # completed_at is volatile and lives in the durable # event log; the file is a WAKE SIGNAL, never truth. - _stable = {k: event.get(k) for k in - ("provider", "event_type", "run_ref", - "sha", "exit_code") if event.get(k) is not None} + _stable = { + k: event.get(k) + for k in ("provider", "event_type", "run_ref", "sha", "exit_code") + if event.get(k) is not None + } completion_file.write_completion(_op_id, _stable) except Exception as e: # noqa: BLE001 — delivery best-effort; wait-record transition + watchdog recovery still cover the lost file - log(f"process-event {eid}: completion-file write " - f"failed: {e}") + log(f"process-event {eid}: completion-file write failed: {e}") matched = 0 for rec in waitstore.pending(): if rec["status"] == "WAITING_EXTERNAL" and waitstore.match(rec, event): @@ -658,14 +676,12 @@ class Handler(BaseHTTPRequestHandler): # single-flight lease + this guard together make resume # idempotent. for rec in waitstore.pending(): - if rec["status"] == "READY_TO_RESUME" \ - and rec.get("resume_cmd") \ - and rec.get("resume_attempts", 0) == 0: - threading.Thread(target=resume, args=(rec,), daemon=True, - name=f"resume-{rec['wait_id']}").start() + if rec["status"] == "READY_TO_RESUME" and rec.get("resume_cmd") and rec.get("resume_attempts", 0) == 0: + threading.Thread(target=resume, args=(rec,), daemon=True, name=f"resume-{rec['wait_id']}").start() matched += 1 try: - self.send_response(200); self.end_headers() + self.send_response(200) + self.end_headers() self.wfile.write(json.dumps({"event_id": eid, "matched": matched}).encode()) except (BrokenPipeError, ConnectionResetError): pass @@ -676,7 +692,9 @@ class Handler(BaseHTTPRequestHandler): try: payload = json.loads(body) if body else {} except json.JSONDecodeError: - self.send_response(400); self.end_headers(); return + self.send_response(400) + self.end_headers() + return event = { "provider": "gitea-local-runner", "task_id": str(payload.get("task_id", "")) or None, @@ -690,24 +708,30 @@ class Handler(BaseHTTPRequestHandler): # fire-and-forget (curl -m 2) — a broken pipe on write is # harmless and must not propagate. try: - threading.Thread(target=wake_sweep, args=(event, eid), daemon=True, - name=f"wake-{payload.get('task_id','?')}").start() + threading.Thread( + target=wake_sweep, args=(event, eid), daemon=True, name=f"wake-{payload.get('task_id', '?')}" + ).start() except Exception as e: log(f"wake spawn error: {e}") try: - self.send_response(200); self.end_headers() + self.send_response(200) + self.end_headers() self.wfile.write(json.dumps({"queued": True, "event_id": eid}).encode()) except (BrokenPipeError, ConnectionResetError): pass return if self.path != "/webhook": - self.send_response(404); self.end_headers(); return + self.send_response(404) + self.end_headers() + return length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length) try: payload = json.loads(body) except json.JSONDecodeError: - self.send_response(400); self.end_headers(); return + self.send_response(400) + self.end_headers() + return action = payload.get("action") or payload.get("event_type") or "" wf = payload.get("workflow_run") or {} event = { @@ -724,15 +748,18 @@ class Handler(BaseHTTPRequestHandler): if rec["status"] == "WAITING_EXTERNAL" and waitstore.match(rec, event): matched += 1 try_transition(rec) - self.send_response(200); self.end_headers() + self.send_response(200) + self.end_headers() self.wfile.write(json.dumps({"event_id": eid, "matched": matched}).encode()) def do_GET(self): if self.path == "/health": - self.send_response(200); self.end_headers() + self.send_response(200) + self.end_headers() self.wfile.write(b'{"ok":true}') return - self.send_response(404); self.end_headers() + self.send_response(404) + self.end_headers() def log_message(self, *a): pass diff --git a/bridge/daemon/verify.py b/bridge/daemon/verify.py index 6881e68e..69837dd6 100755 --- a/bridge/daemon/verify.py +++ b/bridge/daemon/verify.py @@ -6,6 +6,7 @@ references. Returns (verdict, evidence). verdict ∈ {success, failure, pending, unknown}. Only this module may move a wait out of WAITING_EXTERNAL. """ + from __future__ import annotations import json, subprocess from pathlib import Path @@ -13,37 +14,51 @@ from pathlib import Path HOST = "https://gitea.thingscouldgetdicey.online" REPO = "conductor/rune" + def _token() -> str: # Canonical ladder without importing Rune: env → token file → git store import os + tok = os.environ.get("GITEA_TOKEN", "").strip() - if tok: return tok + if tok: + return tok tf = os.path.expanduser("~/.config/opencode/secrets/gitea-token") - if os.path.isfile(tf): return open(tf).read().strip() - r = subprocess.run(["git", "credential", "fill"], + if os.path.isfile(tf): + return open(tf).read().strip() + r = subprocess.run( + ["git", "credential", "fill"], input="protocol=https\nhost=gitea.thingscouldgetdicey.online\nusername=conductor\n\n", - capture_output=True, text=True, timeout=10) + capture_output=True, + text=True, + timeout=10, + ) for line in r.stdout.splitlines(): - if line.startswith("password="): return line[9:] + if line.startswith("password="): + return line[9:] raise RuntimeError("no gitea credential available") + def _api(path: str) -> tuple[int, object]: import urllib.request - req = urllib.request.Request(f"{HOST}/api/v1/repos/{REPO}{path}", - headers={"Authorization": f"token {_token()}"}) + + req = urllib.request.Request(f"{HOST}/api/v1/repos/{REPO}{path}", headers={"Authorization": f"token {_token()}"}) try: with urllib.request.urlopen(req, timeout=20) as r: return r.status, json.loads(r.read()) except urllib.error.HTTPError as e: return e.code, {"error": e.read().decode()[:200]} + def _jobs_list(run_ref: str): """GET jobs for a run, unwrapping the {'jobs': [...]} envelope.""" s, data = _api(f"/actions/runs/{run_ref}/jobs") - if s != 200: return s, None - if isinstance(data, dict): data = data.get("jobs", []) + if s != 200: + return s, None + if isinstance(data, dict): + data = data.get("jobs", []) return s, data if isinstance(data, list) else None + def _resolve_run_id(run_ref: str, sha: str) -> str: """Task-list ids are NOT run ids. If run_ref lookup 404s, resolve the real run id by head SHA via /actions/runs (T15: run 6907 vs task 15743).""" @@ -66,6 +81,7 @@ def _resolve_run_id(run_ref: str, sha: str) -> str: return fallback return "" + def verify_run(run_ref: str, sha: str, expected: str) -> tuple[str, dict]: """run_ref = actions run id (or task id — resolved via SHA lookup). @@ -83,7 +99,7 @@ def verify_run(run_ref: str, sha: str, expected: str) -> tuple[str, dict]: conclusions = {j.get("name"): j.get("conclusion") for j in jobs} statuses = {j.get("status") for j in jobs} if statuses <= {"completed"}: - verdict = "success" if set(conclusions.values()) <= {"success","skipped"} else "failure" + verdict = "success" if set(conclusions.values()) <= {"success", "skipped"} else "failure" return verdict, {"run_ref": run_ref, "jobs": conclusions} return "pending", {"run_ref": run_ref, "statuses": sorted(statuses)} if sha: @@ -91,19 +107,28 @@ def verify_run(run_ref: str, sha: str, expected: str) -> tuple[str, dict]: if s == 200 and isinstance(st, dict): state = st.get("state", "unknown") # success|error|failure|pending|warning rid = _resolve_run_id("", sha) # attach run id so failure evidence works - if state == "success": return "success", {"sha": sha, "state": state, "run_ref": rid} - if state in ("error", "failure"): return "failure", {"sha": sha, "state": state, "run_ref": rid} + if state == "success": + return "success", {"sha": sha, "state": state, "run_ref": rid} + if state in ("error", "failure"): + return "failure", {"sha": sha, "state": state, "run_ref": rid} return "pending", {"sha": sha, "state": state} return "unknown", {"run_ref": run_ref, "sha": sha} + def failed_job_evidence(run_ref: str, sha: str = "") -> dict: """For failure continuations: exact failing job + conclusion.""" run_ref = _resolve_run_id(run_ref, sha) - if not run_ref: return {} + if not run_ref: + return {} s, jobs = _jobs_list(run_ref) - if s != 200 or not isinstance(jobs, list): return {} + if s != 200 or not isinstance(jobs, list): + return {} bad = [j for j in jobs if j.get("conclusion") not in ("success", "skipped", None)] - return {"failed_jobs": [{ "name": j.get("name"), "conclusion": j.get("conclusion"), "url": j.get("url","")} for j in bad]} + return { + "failed_jobs": [ + {"name": j.get("name"), "conclusion": j.get("conclusion"), "url": j.get("url", "")} for j in bad + ] + } # ── RELEASE_DEPLOY verification (W/R v0.1, overnight handoff Part 2) ────── @@ -114,10 +139,11 @@ def failed_job_evidence(run_ref: str, sha: str = "") -> dict: # source SHA → source runs terminal → release successor commit (descendant) # → successor runs success → deployed/current identity converges. + def _git(args: list[str], repo_root: str) -> str: import subprocess - r = subprocess.run(["git"] + args, capture_output=True, text=True, - cwd=repo_root or None, timeout=15) + + r = subprocess.run(["git"] + args, capture_output=True, text=True, cwd=repo_root or None, timeout=15) if r.returncode != 0: raise RuntimeError(f"git {' '.join(args[:2])}: {r.stderr.strip()[:120]}") return r.stdout @@ -147,8 +173,7 @@ def _sha_verdict_from_runs(runs: list[dict]) -> tuple[str, dict]: return "success", {"runs": conclusions} -def verify_release_deploy(source_sha: str, repo_root: str = "", - deployed_link: str = "") -> tuple[str, dict]: +def verify_release_deploy(source_sha: str, repo_root: str = "", deployed_link: str = "") -> tuple[str, dict]: """RELEASE_DEPLOY authority chain. Returns (verdict, evidence).""" source_sha = (source_sha or "")[:12] ev: dict = {"source_sha": source_sha, "kind": "RELEASE_DEPLOY"} @@ -164,9 +189,17 @@ def verify_release_deploy(source_sha: str, repo_root: str = "", if not repo_root: return "pending", {**ev, "note": "no repo_root — cannot check lineage"} try: - out = _git(["log", "--format=%H %s", "--reverse", - f"{source_sha}..master", "--grep=^chore\\(release\\)", - "--extended-regexp"], repo_root) + out = _git( + [ + "log", + "--format=%H %s", + "--reverse", + f"{source_sha}..master", + "--grep=^chore\\(release\\)", + "--extended-regexp", + ], + repo_root, + ) except RuntimeError as e: return "pending", {**ev, "note": str(e)} successors = [l for l in out.splitlines() if l.strip()] @@ -206,8 +239,7 @@ def verify_release_deploy(source_sha: str, repo_root: str = "", still = [r for r in _runs_for_sha(succ_sha) if r.get("status") != "completed"] if still: return "pending", {**ev, "note": "deploy propagating (runs in flight)"} - return "failure", {**ev, - "note": f"deployed {deployed_id} != successor {succ_sha} (all runs terminal)"} + return "failure", {**ev, "note": f"deployed {deployed_id} != successor {succ_sha} (all runs terminal)"} else: return "failure", {**ev, "note": f"deployed link missing: {deployed_link}"} diff --git a/bridge/daemon/waitstore.py b/bridge/daemon/waitstore.py index dbdd0041..e591583d 100755 --- a/bridge/daemon/waitstore.py +++ b/bridge/daemon/waitstore.py @@ -16,6 +16,7 @@ Waits persist as waits/.json (atomic tmp+rename). Leases persist as leases/.lease.json (atomic tmp+rename, validated against /proc so a dead or PID-reused owner is stealable). """ + from __future__ import annotations import fcntl, json, os, tempfile, time, uuid from pathlib import Path @@ -27,8 +28,15 @@ LEASES = ROOT / "leases" SCHEDULABLE = {"READY_TO_RESUME", "READY_TO_RETRY"} VALID_STATES = { - "RUNNABLE", "WAITING_EXTERNAL", "READY_TO_RESUME", "RESUMING", - "COMPLETED", "FAILED", "READY_TO_RETRY", "RESUME_UNVERIFIED", "TIMED_OUT", + "RUNNABLE", + "WAITING_EXTERNAL", + "READY_TO_RESUME", + "RESUMING", + "COMPLETED", + "FAILED", + "READY_TO_RETRY", + "RESUME_UNVERIFIED", + "TIMED_OUT", } TERMINAL = {"COMPLETED", "FAILED", "RESUME_UNVERIFIED", "TIMED_OUT"} @@ -56,32 +64,53 @@ def now() -> str: return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) -def new_wait(*, session_id: str, provider: str, event_type: str, repo: str, - sha: str = "", run_ref: str = "", expected: str = "success", - wait_kind: str = "EXACT_WORKFLOW", target_kind: str = "interactive", - repo_root: str = "", deployed_link: str = "", - objective: str = "", waited_for: str = "", verify_when_woken: str = "", - on_success: str = "", on_failure: str = "", - resume_cmd: list | None = None) -> dict: +def new_wait( + *, + session_id: str, + provider: str, + event_type: str, + repo: str, + sha: str = "", + run_ref: str = "", + expected: str = "success", + wait_kind: str = "EXACT_WORKFLOW", + target_kind: str = "interactive", + repo_root: str = "", + deployed_link: str = "", + objective: str = "", + waited_for: str = "", + verify_when_woken: str = "", + on_success: str = "", + on_failure: str = "", + resume_cmd: list | None = None, +) -> dict: wid = f"wr-{int(time.time())}-{uuid.uuid4().hex[:6]}" rec = { - "wait_id": wid, "session_id": session_id, + "wait_id": wid, + "session_id": session_id, "created_at": now(), - "provider": provider, "event_type": event_type, "repo": repo, - "sha": sha, "run_ref": run_ref, "expected": expected, + "provider": provider, + "event_type": event_type, + "repo": repo, + "sha": sha, + "run_ref": run_ref, + "expected": expected, # T15 §15D: PROCESS_EVENT — a wait whose authoritative verdict is # the matching process-exit event itself (exit_code carried on the # event), e.g. a builder/controller process completing. The event # IS the authority here (the notifier wrapper observed the real # process exit); no Gitea lookup applies. - "wait_kind": wait_kind if wait_kind in ("EXACT_WORKFLOW", "RELEASE_DEPLOY", "PROCESS_EVENT") else "EXACT_WORKFLOW", + "wait_kind": wait_kind + if wait_kind in ("EXACT_WORKFLOW", "RELEASE_DEPLOY", "PROCESS_EVENT") + else "EXACT_WORKFLOW", # CONTENTION POLICY: "interactive" (default, conservative) never # auto-dispatches — MiMo exposes NO supported busy/idle session # signal, so injecting into a possibly-busy interactive session is # NOT YET SAFE. Only explicitly declared headless/disposable # targets ("headless") are auto-resumed. "target_kind": target_kind if target_kind in ("headless", "interactive") else "interactive", - "repo_root": repo_root, "deployed_link": deployed_link, + "repo_root": repo_root, + "deployed_link": deployed_link, # T15 §15D: per-wait resume command override (placeholder template, # same substitution as WR_RESUME_CMD: {session_id}/{msg}/{wait_id}). # A synthetic controller session (e.g. a Scaffolding stage driver) @@ -90,12 +119,15 @@ def new_wait(*, session_id: str, provider: str, event_type: str, repo: str, # in T12-T14. Per-wait override makes ANY registered continuation # target resumable through the same bridge. "resume_cmd": list(resume_cmd) if resume_cmd else None, - "status": "WAITING_EXTERNAL", "last_verified": "", + "status": "WAITING_EXTERNAL", + "last_verified": "", "resume_attempts": 0, "continuation": { - "objective": objective, "waited_for": waited_for, + "objective": objective, + "waited_for": waited_for, "verify_when_woken": verify_when_woken, - "on_success": on_success, "on_failure": on_failure, + "on_success": on_success, + "on_failure": on_failure, }, } _atomic_write(WAITS / f"{wid}.json", rec) @@ -124,8 +156,7 @@ def all_waits() -> list[dict]: def pending() -> list[dict]: - return [r for r in all_waits() - if r.get("status") in {"WAITING_EXTERNAL"} | SCHEDULABLE] + return [r for r in all_waits() if r.get("status") in {"WAITING_EXTERNAL"} | SCHEDULABLE] def schedulable() -> list[dict]: @@ -148,8 +179,10 @@ def requeue(wid: str, note: str) -> dict | None: # ── events (wake signals, idempotent) ───────────────────────────────────── + def record_event(event: dict) -> str: import hashlib + body = json.dumps(event, sort_keys=True) eid = hashlib.sha256(body.encode()).hexdigest()[:16] f = EVENTS / f"{eid}.json" @@ -216,6 +249,7 @@ def lease_path(session_id: str) -> Path: # ids that sanitize identically. import hashlib as _hashlib import re as _re + safe = _re.sub(r"[^A-Za-z0-9._-]+", "_", session_id)[:80] digest = _hashlib.sha256(session_id.encode()).hexdigest()[:10] return LEASES / f"{safe}-{digest}.lease.json" @@ -268,10 +302,13 @@ def acquire_lease(session_id: str, wait_id: str, pid: int, pgid: int) -> dict | os.close(fd) return None # live worker owns this session lease = { - "session_id": session_id, "wait_id": wait_id, - "pid": pid, "pgid": pgid, + "session_id": session_id, + "wait_id": wait_id, + "pid": pid, + "pgid": pgid, "pid_starttime": _pid_starttime(pid), - "acquired_at": now(), "state": "RESUMING", + "acquired_at": now(), + "state": "RESUMING", } _write_lease_locked(session_id, lease) return lease diff --git a/requirements-ci.txt b/requirements-ci.txt index 900270ff..23d21119 100644 --- a/requirements-ci.txt +++ b/requirements-ci.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile pyproject.toml -o /tmp/req3.txt --extra dev --extra memory --python-version 3.11 +# uv pip compile pyproject.toml -o /tmp/req_new.txt --extra dev --extra memory --python-version 3.11 aiohappyeyeballs==2.7.1 # via aiohttp aiohttp==3.14.3 @@ -64,7 +64,7 @@ nodeenv==1.10.0 # via pyright packaging==26.3 # via pytest -platformdirs==4.11.10 +platformdirs==4.11.11 # via textual pluggy==1.6.0 # via diff --git a/scripts/install_wr_daemon.py b/scripts/install_wr_daemon.py index 341b1426..94c0476d 100644 --- a/scripts/install_wr_daemon.py +++ b/scripts/install_wr_daemon.py @@ -11,6 +11,7 @@ way to move code between them: Never hand-edit the installed copy: edit the repo source, install, restart. """ + from __future__ import annotations import argparse @@ -32,10 +33,8 @@ def sha256(p: Path) -> str: def main() -> int: ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--check", action="store_true", - help="verify convergence only; no writes") - ap.add_argument("--restart", action="store_true", - help="restart the daemon after install") + ap.add_argument("--check", action="store_true", help="verify convergence only; no writes") + ap.add_argument("--restart", action="store_true", help="restart the daemon after install") args = ap.parse_args() if not CANONICAL.is_dir(): @@ -73,8 +72,7 @@ def main() -> int: print("CONVERGED: installed copy == canonical repo source") if args.restart: - rc = subprocess.call([sys.executable, - str(REPO / "scripts" / "start_wr_daemon.py")]) + rc = subprocess.call([sys.executable, str(REPO / "scripts" / "start_wr_daemon.py")]) if rc != 0: print("restart failed") return 1 diff --git a/src/rune/event_wait.py b/src/rune/event_wait.py index c07a0635..26b631d6 100644 --- a/src/rune/event_wait.py +++ b/src/rune/event_wait.py @@ -251,7 +251,12 @@ def register_process_wait(operation_id: str, *, contract_id: str = "", or int(os.environ.get("WR_WATCH_SECS", "600")), "continuation_state": continuation_state, "op_type": op_type, "authority": authority, - "task_id": task_id} + "task_id": task_id, + # §3 delivery-domain marker: the daemon drive loop skips + # waits marked "self" (the waiter owns its continuation); + # records WITHOUT the marker remain eligible for the + # env-template session-delivery path (WR_RESUME_CMD). + "resume_mode": resume_mode} ws.save(rec) # cross-user bridge (T16 §16C live finding): the daemon reconciler may # run under a different account than the registering controller — the diff --git a/store/scaffold/lost_event_recovery_spec.json b/store/scaffold/lost_event_recovery_spec.json new file mode 100644 index 00000000..93d35968 --- /dev/null +++ b/store/scaffold/lost_event_recovery_spec.json @@ -0,0 +1,35 @@ +{ + "tranche": "REAL_ASYNC_BRIDGE", + "test_class": "FAILURE-RECOVERY (deliberately separate from healthy-path statistics)", + "status": "SPECIFIED_NOT_EXECUTED", + "principle": "suppressing normal event delivery must NOT silently lose the operation; watchdog/reconciliation is failure detection and must recover terminal truth safely", + "setup": { + "environment": "isolated WR_ROOT (test_wr_v04 pattern) OR live daemon with injected fault \u2014 decide at execution", + "operation": "one real or stub operation with a known terminal authority verdict", + "fault_injection": "suppress ALL normal delivery paths for the operation: block/redirect the notify producer AND the daemon /process-event completion-file write for that operation_id (e.g., fault shim on completion_file.write_completion + notify POST)" + }, + "required_sequence": [ + "waiter parks (WAITING_EXTERNAL, inotify subscribed)", + "source operation reaches terminal truth (authority would say success) \u2014 delivery suppressed", + "watchdog window expires with NO event delivered", + "watchdog/reconciler notices abnormality (no event + authority reread)", + "authority is consulted directly and returns TERMINAL verdict", + "waiter transitions to terminal truth (COMPLETED) with mode=watchdog_authority_recovered", + "durable operation state stamped authority_verified_terminal \u2014 truthful", + "no silent loss: operation did NOT stay WAITING_EXTERNAL past recovery", + "late/duplicate delivery after recovery is idempotent (no double-processing)" + ], + "metrics_labels": { + "FAILURE_RECOVERY_WATCHDOG_CYCLES": "count (>=1 expected)", + "FAILURE_RECOVERY_AUTHORITY_REREAD_COUNT": "count (>=1 expected)", + "FAILURE_RECOVERY_TIME_TO_TERMINAL_TRUTH": "watchdog expiry -> authority verdict", + "FAILURE_RECOVERY_SILENT_LOSS": "must be 0", + "FAILURE_RECOVERY_STATE_CONTRADICTION": "must be 0" + }, + "explicitly_not_claimed": "this test does NOT measure or validate healthy event-driven completion latency \u2014 that is the accepted final unobserved run's domain", + "acceptance": "all FAILURE_RECOVERY_* metrics recorded; SILENT_LOSS=0; STATE_CONTRADICTION=0; terminal truth recovered before deadline containment", + "related_gates": [ + "R8_LOST_EVENT_RECOVERY" + ], + "created_at": "2026-09-19T01:20:00Z" +} diff --git a/store/scaffold/mimo_semantics_gates.json b/store/scaffold/mimo_semantics_gates.json index 52f399e5..0f3011be 100644 --- a/store/scaffold/mimo_semantics_gates.json +++ b/store/scaffold/mimo_semantics_gates.json @@ -1,152 +1,152 @@ { - "tranche": "MIMO_EXECUTION_SEMANTICS", - "purpose": "Fix the caller, freeze Rune core \u2014 intent compilation at the MiMo execution boundary", - "predecessor": "event_hardening_gates.json (superseded verdict)", - "created_at": "2026-09-18T19:57:25Z", - "gates": [ - { - "gate_id": "M10_LIVE_MASTER_SINGLE_CONTINUATION", - "mandatory": true, - "requirement": "master/deploy resolved via one canonical continuation; no model re-arms", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M11_RUNE_CORE_FROZEN", - "mandatory": true, - "requirement": "RUNE_CORE_FILES_MODIFIED=0 or non-MiMo reproduction proven for each", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M12_CLEAN_WINDOW_AUDIT", - "mandatory": true, - "requirement": "no manual internal APIs in the acceptance window", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M1_CALLER_ROOT_CAUSE_IDENTIFIED", - "mandatory": true, - "requirement": "the exact seam where MiMo executes mechanical wait policy instead of compiling intent", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M2_OPERATION_ID_FIRST_TOOL_SCHEMA", - "mandatory": true, - "requirement": "await-operation tool: operation_id-first; legacy args accepted as hints only", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M3_TASK_BOUND_OPERATION_RESOLUTION", - "mandatory": true, - "requirement": "deterministic task->operation binding; model sha/op_type/deadline non-authoritative", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M4_BASH_INTENT_COMPILATION_ACTIVE", - "mandatory": true, - "requirement": "bash.ts routes external wait intent to canonical continuation; literal sleep suppressed", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M5_LOWER_LEVEL_INTERNALS_EXCLUDED", - "mandatory": true, - "requirement": "model cannot select ci_event_wait/deploy_qualify/authority helpers as strategies", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M6_OBSERVABILITY_COMPACT_LOG", - "mandatory": true, - "requirement": "every translation recorded with binding source/provenance", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M7_LIVE_WRONG_ARGS_PROOF", - "mandatory": true, - "requirement": "model emits wrong deadline/op_type/sha; harness compiles to bound operation", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M8_LIVE_BASH_SLEEP_SUPPRESSED", - "mandatory": true, - "requirement": "bash sleep intent compiled; LITERAL_EXTERNAL_SLEEP_EXECUTED=false", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - }, - { - "gate_id": "M9_LIVE_PR_SINGLE_CONTINUATION", - "mandatory": true, - "requirement": "PR checks resolved via one canonical continuation; no model re-arms", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T19:57:25Z", - "resolved_at": "" - } - ] + "tranche": "MIMO_EXECUTION_SEMANTICS", + "purpose": "Fix the caller, freeze Rune core \u2014 intent compilation at the MiMo execution boundary", + "predecessor": "event_hardening_gates.json (superseded verdict)", + "created_at": "2026-09-18T19:57:25Z", + "gates": [ + { + "gate_id": "M10_LIVE_MASTER_SINGLE_CONTINUATION", + "mandatory": true, + "requirement": "master/deploy resolved via one canonical continuation; no model re-arms", + "status": "UNRUN", + "evidence": "master/deploy flow via model-facing tool not exercised in this tranche", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "" + }, + { + "gate_id": "M11_RUNE_CORE_FROZEN", + "mandatory": true, + "requirement": "RUNE_CORE_FILES_MODIFIED=0 or non-MiMo reproduction proven for each", + "status": "PASS", + "evidence": "RUNE core touch = src/rune/event_wait.py resume_mode (additive, default-preserving) with frozen non-MiMo tests; all other core files unmodified in tranche", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M12_CLEAN_WINDOW_AUDIT", + "mandatory": true, + "requirement": "no manual internal APIs in the acceptance window", + "status": "PASS", + "evidence": "final unobserved run window 01:01:54..01:10:16Z: zero manual observers, zero cron observers, zero manual internal API calls", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M1_CALLER_ROOT_CAUSE_IDENTIFIED", + "mandatory": true, + "requirement": "the exact seam where MiMo executes mechanical wait policy instead of compiling intent", + "status": "PASS", + "evidence": "root cause: model emitted mechanical wait policy at the execution boundary; fix = intent normalization (await_intent) + operation-id-first tool + bash routing; live proof final run", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M2_OPERATION_ID_FIRST_TOOL_SCHEMA", + "mandatory": true, + "requirement": "await-operation tool: operation_id-first; legacy args accepted as hints only", + "status": "PASS", + "evidence": "~/.mimocode/tools/await-operation.ts (sha256 894dc91d\u2026): operation_id + task_id first; sha/op_type/deadline optional non-authoritative hints; final run used operation_id only", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M3_TASK_BOUND_OPERATION_RESOLUTION", + "mandatory": true, + "requirement": "deterministic task->operation binding; model sha/op_type/deadline non-authoritative", + "status": "PASS", + "evidence": "--task-id through cli+tool; frozen tests TestTaskBinding (bound resolution, unbound reports missing); stored operation wins over model hints. NOTE: live acceptance run used explicit operation_id; task-scoped live resolution proven at unit level", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M4_BASH_INTENT_COMPILATION_ACTIVE", + "mandatory": true, + "requirement": "bash.ts routes external wait intent to canonical continuation; literal sleep suppressed", + "status": "PASS", + "evidence": "bash.ts (1054d8e2) routes external wait intent via await_intent_cli route-bash; live routing observed 00:47:00Z; historical entry raw_sleep_seconds_suppressed=240", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M5_LOWER_LEVEL_INTERNALS_EXCLUDED", + "mandatory": true, + "requirement": "model cannot select ci_event_wait/deploy_qualify/authority helpers as strategies", + "status": "PASS", + "evidence": "model-facing surface exposes only await-operation/await_intent_cli; ci_event_wait/deploy_qualify not exposed as strategies; final run tool inventory = await-operation only", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M6_OBSERVABILITY_COMPACT_LOG", + "mandatory": true, + "requirement": "every translation recorded with binding source/provenance", + "status": "PASS", + "evidence": "every translation recorded in store/await_intent_log.jsonl with normalized/evidence/matched_by/deadline_provenance fields", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M7_LIVE_WRONG_ARGS_PROOF", + "mandatory": true, + "requirement": "model emits wrong deadline/op_type/sha; harness compiles to bound operation", + "status": "PASS", + "evidence": "historical live evidence 2026-09-18T10:30:40Z: model deadline ignored (model_deadline_ignored=true), 240s sleep suppressed; final run emitted no wrong args (operation_id only) \u2014 hint-ignoring machinery active in both", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M8_LIVE_BASH_SLEEP_SUPPRESSED", + "mandatory": true, + "requirement": "bash sleep intent compiled; LITERAL_EXTERNAL_SLEEP_EXECUTED=false", + "status": "PASS", + "evidence": "historical: raw_sleep_seconds_suppressed=240 with LITERAL_EXTERNAL_SLEEP not executed; live routing of sleep intent observed 00:47:00Z (plain local sleep executes by design; external intent compiles)", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "M9_LIVE_PR_SINGLE_CONTINUATION", + "mandatory": true, + "requirement": "PR checks resolved via one canonical continuation; no model re-arms", + "status": "UNRUN", + "evidence": "PR-checks flow via model-facing tool not exercised in this tranche (branch flow proven; PR/master remain)", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T19:57:25Z", + "resolved_at": "" + } + ] } \ No newline at end of file diff --git a/store/scaffold/real_bridge_diagnostic_run.json b/store/scaffold/real_bridge_diagnostic_run.json new file mode 100644 index 00000000..db1ee173 --- /dev/null +++ b/store/scaffold/real_bridge_diagnostic_run.json @@ -0,0 +1,29 @@ +{ + "tranche": "REAL_ASYNC_BRIDGE", + "run_class": "DIAGNOSTIC_EVIDENCE_ONLY (contaminated by observer: 2 forensic sleeps, 1 armed collector; superseded by final unobserved run)", + "operation_id": "ci-c288f4726d03993bd6c8cc751c9ab0cdb8713057", + "chain": { + "session_launch_utc": "2026-09-19T00:41:36Z", + "await_tool_called_at": "2026-09-19T00:42:06Z", + "waiter_registered_at": "2026-09-19T00:42:06Z", + "resume_mode": "self (resume_cmd=None)", + "source_terminal_at": "2026-09-19T00:46:20.94Z", + "completion_event_received_at": "2026-09-19T00:46:20Z", + "completion_file_written_at": "2026-09-19T00:46:20Z", + "inotify_wake_at": "2026-09-19T00:46:20Z (envelope event_received_at)", + "authority_verified_at": "2026-09-19T00:47:38Z", + "await_tool_returned_at": "2026-09-19T00:47:38.741Z (transcript tool_use state=completed)", + "next_model_turn_at": "2026-09-19T00:48:21.657Z (automatic)", + "session_exit": "natural, before 00:59:52Z" + }, + "canonical_operation_state": { + "status": "COMPLETED", + "verdict": "success", + "evidence_kind": "authority_verified_terminal", + "mode": "event" + }, + "state_ownership_fix_held": true, + "internal_rearm_count": 1, + "watchdog_wake": 0, + "deadline_wake": 0 +} diff --git a/store/scaffold/real_bridge_final_acceptance.json b/store/scaffold/real_bridge_final_acceptance.json new file mode 100644 index 00000000..732d27fd --- /dev/null +++ b/store/scaffold/real_bridge_final_acceptance.json @@ -0,0 +1,163 @@ +{ + "tranche": "REAL_ASYNC_BRIDGE", + "artifact_class": "FINAL_UNOBSERVED_ACCEPTANCE_RUN \u2014 canonical healthy-path evidence", + "verdict": "REAL_ASYNC_BRIDGE_COMPLETE", + "run_identity": { + "sha": "5b61f8a8bb8273d0fbe5124dd81a952f208ef41c", + "operation_id": "ci-5b61f8a8bb8273d0fbe5124dd81a952f208ef41c", + "operation_created_at": "2026-09-19T01:01:36Z", + "session_launch_at": "2026-09-19T01:01:54Z", + "session_pid": 3137854, + "wait_id": "wr-1789779758-4cb2a6" + }, + "raw_timeline": [ + { + "event": "operation_created_at", + "at": "2026-09-19T01:01:36Z", + "source": "push record (pre-launch)" + }, + { + "event": "session_launch", + "at": "2026-09-19T01:01:54Z", + "source": "launch record" + }, + { + "event": "model_tool_call_at", + "at": "2026-09-19T01:02:31.165Z", + "source": "transcript step_start" + }, + { + "event": "normalize (tool internals)", + "at": "2026-09-19T01:02:37Z", + "source": "store/await_intent_log.jsonl evidence=explicit_id_fresh_registration" + }, + { + "event": "waiter_registered_at", + "at": "2026-09-19T01:02:38Z", + "source": "wait record created_at" + }, + { + "event": "watchdog_failure_detector_cycle", + "at": "~2026-09-19T01:07:38Z", + "source": "envelope rearm_reason (park+300s)", + "precision": "SUPPORTED_BUT_COARSE" + }, + { + "event": "source_terminal_at", + "at": "2026-09-19T01:08:56.59Z", + "source": "event payload completed_at (exit_code 0)" + }, + { + "event": "completion_event_received_at", + "at": "2026-09-19T01:08:56Z", + "source": "durable event d6ebf9466ebbd776 received_at" + }, + { + "event": "completion_file_written_at", + "at": "2026-09-19T01:08:56Z", + "source": "resumes/ci-5b61f8a\u2026.done", + "precision": "FILESYSTEM_MTIME_EVIDENCE" + }, + { + "event": "inotify_wake_at", + "at": "2026-09-19T01:08:56Z", + "source": "envelope event_received_at + controller_resumed_at (the event that released the blocking wait)", + "precision": "second" + }, + { + "event": "authority_verified_at", + "at": "2026-09-19T01:09:58Z", + "source": "envelope verdict_at" + }, + { + "event": "await_tool_returned_at", + "at": "2026-09-19T01:09:58.397Z", + "source": "transcript tool_use state=completed callID=call_d6491c1a337b44f89c41dae1" + }, + { + "event": "next_model_turn_at", + "at": "2026-09-19T01:10:12.026Z", + "source": "transcript step_start (automatic)" + }, + { + "event": "CLEAN_PROOF_COMPLETE_at", + "at": "2026-09-19T01:10:16.102Z", + "source": "transcript text" + }, + { + "event": "session_exit_at", + "at": "2026-09-19T01:10:16Z", + "source": "/tmp/final_exit.txt exit_rc=0 (wrapper-written)", + "precision": "second" + } + ], + "model_call_counts": { + "AWAIT_OPERATION_CALL_COUNT": 1, + "OTHER_MODEL_TOOL_CALL_COUNT": 0, + "MODEL_SHELL_COUNT": 0, + "MODEL_SLEEP_COUNT": 0, + "MODEL_POLL_COUNT": 0, + "MODEL_CRON_COUNT": 0, + "MODEL_REARM_COUNT": 0, + "INTERNAL_EVENT_RESUBSCRIPTION_COUNT": 1 + }, + "wake_classification": { + "mode": "event", + "NORMAL_COMPLETION_DISCOVERED_BY_EVENT": 1, + "NORMAL_COMPLETION_DISCOVERED_BY_WATCHDOG": 0, + "NORMAL_COMPLETION_DISCOVERED_BY_DEADLINE": 0, + "WATCHDOG_FAILURE_DETECTOR_CYCLE_COUNT": 1, + "watchdog_nuance": "the watchdog cycle found TRUTHFUL PENDING (source had not completed) and re-subscribed to the event channel; it did NOT determine completion latency \u2014 the event did, 78s later" + }, + "canonical_terminal_state": { + "status": "COMPLETED", + "verdict": "success", + "resume_evidence_kind": "authority_verified_terminal", + "continuation_state": "RESUMABLE", + "resume_mode": "self", + "resume_cmd": null, + "fifo_contradiction_recurred": false + }, + "observer_contamination": { + "MANUAL_OBSERVER_COUNT": 0, + "CRON_OBSERVER_COUNT": 0, + "window": "2026-09-19T01:01:54Z..01:10:16Z" + }, + "latencies": { + "SOURCE_TO_EVENT": "<=1s (second precision)", + "EVENT_TO_COMPLETION_WRITE": "<=1s (FILESYSTEM_MTIME_EVIDENCE)", + "EVENT_TO_WAKE": "<=1s (second precision)", + "WAKE_TO_AUTHORITY": "62s (Gitea authority API read; no rearm inside gap)", + "AUTHORITY_TO_TOOL_RETURN": "<=0.4s", + "TOOL_RETURN_TO_MODEL_TURN": "13.4s (attribution SOURCE_UNKNOWN: provider inference vs harness scheduling)", + "SOURCE_TO_MODEL_RESUME": "75.4s", + "SOURCE_TO_CLEAN_PROOF_COMPLETE": "79.5s", + "wall_secs_envelope": 440.343 + }, + "acceptance_conditions": { + "one_model_facing_invocation": "PASS", + "quiescent_while_waiting": "PASS", + "no_model_polling_sleep": "PASS", + "real_source_event_durable_delivery": "PASS", + "event_path_wake": "PASS", + "watchdog_not_completion_discoverer": "PASS", + "deadline_not_completion_discoverer": "PASS", + "authority_terminal_truth": "PASS", + "durable_state_agrees_with_authority": "PASS", + "same_invocation_returned": "PASS", + "automatic_next_model_turn": "PASS", + "clean_proof_complete_emitted": "PASS", + "natural_exit": "PASS (rc=0)", + "no_observer_drove_continuation": "PASS" + }, + "duplicate_run_effects": "none \u2014 exactly 1 durable event for this SHA", + "raw_evidence_pointers": { + "transcript": "/tmp/final_transcript.jsonl (session-local; events embedded above are canonical)", + "exit_marker": "/tmp/final_exit.txt", + "wait_record": "/mnt/workdrive/wr-bridge/waits/wr-1789779758-4cb2a6.json", + "durable_event": "/mnt/workdrive/wr-bridge/events/d6ebf9466ebbd776.json", + "completion_file": "/mnt/workdrive/wr-bridge/resumes/ci-5b61f8a8bb8273d0fbe5124dd81a952f208ef41c.done", + "intent_log": "store/await_intent_log.jsonl entry 2026-09-19T01:02:37Z" + }, + "recorded_at": "2026-09-19T01:20:00Z" +} diff --git a/store/scaffold/real_bridge_gates.json b/store/scaffold/real_bridge_gates.json index b9292196..5d9c2ea8 100644 --- a/store/scaffold/real_bridge_gates.json +++ b/store/scaffold/real_bridge_gates.json @@ -1,200 +1,200 @@ { - "tranche": "REAL_ASYNC_BRIDGE", - "purpose": "Real event-driven bridge \u2014 completion-file delivery replaces FIFO, intent compilation at MiMo boundary", - "predecessor": "event_hardening_gates.json (18/18, superseded by clean-window contradictions)", - "created_at": "2026-09-18T21:26:48Z", - "gates": [ - { - "gate_id": "R10_DUPLICATE_EVENT_IDEMPOTENCE", - "mandatory": true, - "requirement": "duplicate events processed once", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R11_LIVE_SHORT_OPERATION", - "mandatory": true, - "requirement": "short operation resumes at actual completion time", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R12_LIVE_VARIABLE_OPERATION", - "mandatory": true, - "requirement": "variable-duration operation resumes at actual completion time", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R13_LIVE_GITEA_FLOWS", - "mandatory": true, - "requirement": "branch/PR/master via tool, one continuation per flow", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R14_STALE_RECORDS_NO_BLOCK", - "mandatory": true, - "requirement": "unrelated stale records don't prevent task-bound continuation", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R15_RUNE_CORE_MUTATION_AUDIT", - "mandatory": true, - "requirement": "Rune core files modified only with non-MiMo reproduction", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R16_CLEAN_WINDOW", - "mandatory": true, - "requirement": "no manual internal APIs in acceptance window", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R1_CALLER_INTENT_COMPILED", - "mandatory": true, - "requirement": "MiMo harness compiles imperfect intent (sleep/check/wait) to ContinuePendingOperation", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R2_OPERATION_ID_BOUND", - "mandatory": true, - "requirement": "task-bound operation resolution works without SHA/type/deadline from model", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R3_COMPLETION_FILE_DELIVERY", - "mandatory": true, - "requirement": "durable completion file replaces FIFO for process-wait delivery", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R4_INOTIFY_WAKE_ZERO_LATENCY", - "mandatory": true, - "requirement": "waiter wakes on file creation via kernel event, not timer", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R5_SOURCE_EVENT_CAUSES_RESUME", - "mandatory": true, - "requirement": "normal completion wakes from source event, not watchdog/deadline", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R6_INTERMEDIATE_EVENT_REBLOCK", - "mandatory": true, - "requirement": "nonterminal authority -> waiter returns to event wait, not timer", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R7_WATCHDOG_FAILURE_DETECTOR_ONLY", - "mandatory": true, - "requirement": "watchdog fires only on genuine infrastructure failure, not as scheduler", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R8_LOST_EVENT_RECOVERY", - "mandatory": true, - "requirement": "dropped event -> watchdog authority reread -> resume before deadline", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - }, - { - "gate_id": "R9_RESTART_RECOVERY", - "mandatory": true, - "requirement": "waiter/bridge restart -> durable state recovered -> continuation resumes", - "status": "UNRUN", - "evidence": "", - "blocker": "", - "waiver_reason": "", - "waived_by": "", - "declared_at": "2026-09-18T21:26:48Z", - "resolved_at": "" - } - ] + "tranche": "REAL_ASYNC_BRIDGE", + "purpose": "Real event-driven bridge \u2014 completion-file delivery replaces FIFO, intent compilation at MiMo boundary", + "predecessor": "event_hardening_gates.json (18/18, superseded by clean-window contradictions)", + "created_at": "2026-09-18T21:26:48Z", + "gates": [ + { + "gate_id": "R10_DUPLICATE_EVENT_IDEMPOTENCE", + "mandatory": true, + "requirement": "duplicate events processed once", + "status": "PASS", + "evidence": "diagnostic run: 2 events for one SHA -> byte-identical completion file rewrite, no double-dispatch; test_wr_v04 R10 byte-identical duplicates; final run: exactly 1 event", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R11_LIVE_SHORT_OPERATION", + "mandatory": true, + "requirement": "short operation resumes at actual completion time", + "status": "PASS", + "evidence": "final unobserved run (accepted per directive 2026-09-19): real branch CI source event -> durable completion -> event wake -> authority-verified success; envelope mode=event verdict=success", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R12_LIVE_VARIABLE_OPERATION", + "mandatory": true, + "requirement": "variable-duration operation resumes at actual completion time", + "status": "PASS", + "evidence": "same accepted run: operation duration 440s was unknown a priori and exceeded one 300s watchdog window; waiter survived the watchdog cycle and resumed at ACTUAL completion (01:08:56 event), not at a timer boundary \u2014 the variable-duration property", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R13_LIVE_GITEA_FLOWS", + "mandatory": true, + "requirement": "branch/PR/master via tool, one continuation per flow", + "status": "PARTIAL", + "evidence": "branch flow PROVEN via model-facing tool (final accepted run). PR-flow and master-flow via tool NOT exercised in this tranche \u2014 remain for a follow-up live pass", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R14_STALE_RECORDS_NO_BLOCK", + "mandatory": true, + "requirement": "unrelated stale records don't prevent task-bound continuation", + "status": "PASS", + "evidence": "final run: 90+ stale WAITING_EXTERNAL records existed; explicit operation_id resolution proceeded with zero interference (no AMBIGUOUS); task-binding tests prove scoped resolution ignores stale globals", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R15_RUNE_CORE_MUTATION_AUDIT", + "mandatory": true, + "requirement": "Rune core files modified only with non-MiMo reproduction", + "status": "PASS", + "evidence": "src/rune/event_wait.py modified (resume_mode param, additive; default preserves T15 semantics) with non-MiMo frozen reproduction tests/test_real_async_integration.py::TestResumeModeSelf", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R16_CLEAN_WINDOW", + "mandatory": true, + "requirement": "no manual internal APIs in acceptance window", + "status": "PASS", + "evidence": "final unobserved run: MANUAL_OBSERVER_COUNT=0, CRON_OBSERVER_COUNT=0 across window 01:01:54..01:10:16Z; no manual internal APIs used", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R1_CALLER_INTENT_COMPILED", + "mandatory": true, + "requirement": "MiMo harness compiles imperfect intent (sleep/check/wait) to ContinuePendingOperation", + "status": "PASS", + "evidence": "await_intent_log 2026-09-18T10:30:40Z: raw_command_kind=sleep_poll raw_sleep_seconds_suppressed=240 -> ContinuePendingOperation (model_deadline_ignored=true); bash.ts route-bash active (hash 1054d8e2); live routing of a sleep command observed 2026-09-19T00:47:00Z (AMBIGUOUS, logged)", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R2_OPERATION_ID_BOUND", + "mandatory": true, + "requirement": "task-bound operation resolution works without SHA/type/deadline from model", + "status": "PASS", + "evidence": "final acceptance run: model passed ONLY operation_id; normalization resolved op_type/authority/deadline from GOVERNED_OPERATION_POLICY (matched_by=explicit_id_fresh_registration); task-binding frozen tests tests/test_real_async_integration.py::TestTaskBinding", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R3_COMPLETION_FILE_DELIVERY", + "mandatory": true, + "requirement": "durable completion file replaces FIFO for process-wait delivery", + "status": "PASS", + "evidence": "final run: durable resumes/ci-5b61f8a\u2026.done written 01:08:56, waiter resume_cmd=None (self-mode), woke from file; test_wr_v04_completion R3 (daemon subprocess, isolated root)", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R4_INOTIFY_WAKE_ZERO_LATENCY", + "mandatory": true, + "requirement": "waiter wakes on file creation via kernel event, not timer", + "status": "PASS", + "evidence": "test_wr_v04_completion: kernel wake latency 7-9ms measured; final run: wake same second as file write (envelope event_received_at=01:08:56)", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R5_SOURCE_EVENT_CAUSES_RESUME", + "mandatory": true, + "requirement": "normal completion wakes from source event, not watchdog/deadline", + "status": "PASS", + "evidence": "final run: completion discovered by EVENT at 01:08:56 (NORMAL_COMPLETION_DISCOVERED_BY_EVENT=1); watchdog cycle at ~01:07:38 found truthful pending; deadline 1800s untouched (wall 440s)", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R6_INTERMEDIATE_EVENT_REBLOCK", + "mandatory": true, + "requirement": "nonterminal authority -> waiter returns to event wait, not timer", + "status": "PASS", + "evidence": "final run: watchdog expiry -> authority pending -> RE-SUBSCRIBED to inotify (not timer), then event wake; diagnostic run: 2 intermediate pending verdicts -> re-subscription, duplicate event re-fired kernel wake", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R7_WATCHDOG_FAILURE_DETECTOR_ONLY", + "mandatory": true, + "requirement": "watchdog fires only on genuine infrastructure failure, not as scheduler", + "status": "PASS", + "evidence": "final run: WATCHDOG_FAILURE_DETECTOR_CYCLE_COUNT=1, NORMAL_COMPLETION_DISCOVERED_BY_WATCHDOG=0 \u2014 watchdog found truthful pending and re-subscribed; completion latency determined by source event (78s after the cycle)", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + }, + { + "gate_id": "R8_LOST_EVENT_RECOVERY", + "mandatory": true, + "requirement": "dropped event -> watchdog authority reread -> resume before deadline", + "status": "UNRUN", + "evidence": "deliberate lost-event fault injection NOT executed (separate FAILURE-RECOVERY test, spec store/scaffold/lost_event_recovery_spec.json). Note: the recovery mechanism's first half (watchdog expiry -> direct authority reread) was exercised live in the final run and returned truthful pending \u2014 but the full dropped-event-to-terminal-recovery proof remains unrun", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "" + }, + { + "gate_id": "R9_RESTART_RECOVERY", + "mandatory": true, + "requirement": "waiter/bridge restart -> durable state recovered -> continuation resumes", + "status": "PASS", + "evidence": "daemon restart 2026-09-19T02:43 local: startup_recovery reconciled orphaned RESUMING records truthfully (RESUME_UNVERIFIED + lease release); test_wr_v04 R9: event-before-waiter recovered from durable file", + "blocker": "", + "waiver_reason": "", + "waived_by": "", + "declared_at": "2026-09-18T21:26:48Z", + "resolved_at": "2026-09-19T01:20:00Z" + } + ] } \ No newline at end of file diff --git a/tests/test_completion_delivery.py b/tests/test_completion_delivery.py index 1bde8332..80846594 100644 --- a/tests/test_completion_delivery.py +++ b/tests/test_completion_delivery.py @@ -8,6 +8,7 @@ detection only, not the delivery path). completion_file.py must resolve the wr root identically under both env conventions (WR_ROOT / RUNE_WR_ROOT) so daemon, waiter, and tests can never disagree about delivery location. """ + from __future__ import annotations import importlib @@ -30,17 +31,16 @@ class CompletionRootResolution(unittest.TestCase): def _bridge(self, env: dict) -> Path: import completion_file + with mock.patch.dict(os.environ, env, clear=False): return completion_file._wr_bridge() def test_rune_wr_root_convention_respected(self): - got = self._bridge({"RUNE_WR_ROOT": "/tmp/rune-root-a", - "WR_ROOT": ""}) + got = self._bridge({"RUNE_WR_ROOT": "/tmp/rune-root-a", "WR_ROOT": ""}) self.assertEqual(got, Path("/tmp/rune-root-a")) def test_wr_root_convention_takes_precedence(self): - got = self._bridge({"RUNE_WR_ROOT": "/tmp/rune-root-a", - "WR_ROOT": "/tmp/wr-root-b"}) + got = self._bridge({"RUNE_WR_ROOT": "/tmp/rune-root-a", "WR_ROOT": "/tmp/wr-root-b"}) self.assertEqual(got, Path("/tmp/wr-root-b")) def test_default_root_unchanged(self): @@ -63,6 +63,7 @@ class NotifyWritesDurableCompletion(unittest.TestCase): "RUNE_EVENT_BRIDGE_URL": "http://127.0.0.1:1", # nothing listens } import notify_event_bridge as neb + with mock.patch.dict(os.environ, env, clear=False): rc = neb.main() self.assertEqual(rc, 0, "notify must never fail CI") @@ -81,6 +82,7 @@ class NotifyWritesDurableCompletion(unittest.TestCase): "RUNE_EVENT_BRIDGE_URL": "http://127.0.0.1:1", } import notify_event_bridge as neb + with mock.patch.dict(os.environ, env, clear=False): rc = neb.main() self.assertEqual(rc, 0, "delivery is best-effort; CI must stay green") diff --git a/tests/test_real_async_integration.py b/tests/test_real_async_integration.py index 4d7a9004..f5d42070 100644 --- a/tests/test_real_async_integration.py +++ b/tests/test_real_async_integration.py @@ -13,6 +13,7 @@ Task binding (§9): - task-bound resolution scopes to the bound operation; stored state wins over model hints """ + from __future__ import annotations import importlib @@ -52,6 +53,7 @@ class _IsolatedRoot(unittest.TestCase): def _waitstore(self): import waitstore + importlib.reload(waitstore) return waitstore @@ -59,22 +61,27 @@ class _IsolatedRoot(unittest.TestCase): # daemon caches waitstore at import; reload AFTER waitstore so its # binding follows this test's isolated root import daemon as wrd + importlib.reload(wrd) return wrd - def _new_wait(self, ws, *, session_id, status, task_id="", - resume_cmd=None, stale_secs=0, sha="ab" * 20, **kw): - rec = ws.new_wait(session_id=session_id, provider="gitea", - event_type="ci_completed", repo="conductor/rune", - sha=sha, run_ref=f"ci-{sha}", - target_kind="headless", **kw) + def _new_wait(self, ws, *, session_id, status, task_id="", resume_cmd=None, stale_secs=0, sha="ab" * 20, **kw): + rec = ws.new_wait( + session_id=session_id, + provider="gitea", + event_type="ci_completed", + repo="conductor/rune", + sha=sha, + run_ref=f"ci-{sha}", + target_kind="headless", + **kw, + ) rec["status"] = status rec["ext"] = {"operation_id": rec["run_ref"], "task_id": task_id} if resume_cmd is not None: rec["resume_cmd"] = resume_cmd if stale_secs: - rec["last_updated"] = time.strftime( - "%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - stale_secs)) + rec["last_updated"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - stale_secs)) ws.save(rec) return rec @@ -84,22 +91,23 @@ class TestResumeModeSelf(_IsolatedRoot): def test_self_mode_registers_no_resume_cmd(self): import importlib as il + os.environ["RUNE_WR_CODE"] = str(DAEMON_SRC) import rune.event_wait as ew + il.reload(ew) - rec = ew.register_process_wait("ci-selftest0123", project="rune", - resume_mode="self") - self.assertIsNone(rec.get("resume_cmd"), - "self-mode must not register a delivery worker") + rec = ew.register_process_wait("ci-selftest0123", project="rune", resume_mode="self") + self.assertIsNone(rec.get("resume_cmd"), "self-mode must not register a delivery worker") def test_default_mode_preserves_fifo_semantics(self): import importlib as il + os.environ["RUNE_WR_CODE"] = str(DAEMON_SRC) import rune.event_wait as ew + il.reload(ew) rec = ew.register_process_wait("ci-fifotest01234") - self.assertIsNotNone(rec.get("resume_cmd"), - "T15 builder semantics must be preserved") + self.assertIsNotNone(rec.get("resume_cmd"), "T15 builder semantics must be preserved") class TestTerminalGuard(_IsolatedRoot): @@ -108,19 +116,16 @@ class TestTerminalGuard(_IsolatedRoot): def test_resume_skips_terminal_wait(self): ws = self._waitstore() wrd = self._daemon() - rec = self._new_wait(ws, session_id="s-term", status="COMPLETED", - resume_cmd=["/bin/true"]) + rec = self._new_wait(ws, session_id="s-term", status="COMPLETED", resume_cmd=["/bin/true"]) rec["verdict"] = "success" ws.save(rec) wrd.resume(rec) fresh = ws.load(rec["wait_id"]) - self.assertEqual(fresh["status"], "COMPLETED", - "terminal state was downgraded by delivery path") + self.assertEqual(fresh["status"], "COMPLETED", "terminal state was downgraded by delivery path") leases = list((self.root / "leases").glob("*.json")) self.assertEqual(leases, [], "lease created for terminal wait") - class TestOrphanSweep(_IsolatedRoot): """§4: bounded mid-run reconciliation of orphaned resume flows.""" @@ -128,19 +133,22 @@ class TestOrphanSweep(_IsolatedRoot): ws = self._waitstore() wrd = self._daemon() wrd.ORPHAN_GRACE_SECS = 5 # deterministic, small - rec = self._new_wait(ws, session_id="s-orphan", status="RESUMING", - resume_cmd=["/bin/true"]) + rec = self._new_wait(ws, session_id="s-orphan", status="RESUMING", resume_cmd=["/bin/true"]) rec["resume_attempts"] = 1 ws.save(rec) # backdate AFTER save: save() stamps last_updated=now - stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", - time.gmtime(time.time() - 60)) + stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 60)) rec["last_updated"] = stale (ws.WAITS / f"{rec['wait_id']}.json").write_text(json.dumps(rec)) # dead-owner lease - lease = {"session_id": "s-orphan", "wait_id": rec["wait_id"], - "pid": 2**22, "pgid": 2**22, "pid_starttime": "", - "acquired_at": "2020-01-01T00:00:00Z"} + lease = { + "session_id": "s-orphan", + "wait_id": rec["wait_id"], + "pid": 2**22, + "pgid": 2**22, + "pid_starttime": "", + "acquired_at": "2020-01-01T00:00:00Z", + } ws.lease_path("s-orphan").write_text(json.dumps(lease)) healed = wrd.orphan_sweep() self.assertEqual(healed, 1) @@ -153,13 +161,13 @@ class TestOrphanSweep(_IsolatedRoot): ws = self._waitstore() wrd = self._daemon() wrd.ORPHAN_GRACE_SECS = 3600 - rec = self._new_wait(ws, session_id="s-live", status="RESUMING", - resume_cmd=["/bin/true"]) + rec = self._new_wait(ws, session_id="s-live", status="RESUMING", resume_cmd=["/bin/true"]) self.assertEqual(wrd.orphan_sweep(), 0) self.assertEqual(ws.load(rec["wait_id"])["status"], "RESUMING") def _waitstore(self): import waitstore + importlib.reload(waitstore) return waitstore @@ -169,25 +177,25 @@ class TestCliContract(unittest.TestCase): def test_unknown_flag_rejected_loudly(self): import ci_event_wait as cew - with mock.patch.object(sys, "argv", - ["ci_event_wait.py", "--sha", "a" * 40, - "--watchdoog", "5"]): + + with mock.patch.object(sys, "argv", ["ci_event_wait.py", "--sha", "a" * 40, "--watchdoog", "5"]): rc = cew.main() self.assertEqual(rc, 2, "unknown flag must exit 2, not be ignored") def test_positional_rejected_loudly(self): import ci_event_wait as cew - with mock.patch.object(sys, "argv", - ["ci_event_wait.py", "extra"]): + + with mock.patch.object(sys, "argv", ["ci_event_wait.py", "extra"]): rc = cew.main() self.assertEqual(rc, 2) def test_watchdog_flag_reaches_run(self): import ci_event_wait as cew - with mock.patch.object(cew, "run", return_value=0) as run_mock, \ - mock.patch.object(sys, "argv", - ["ci_event_wait.py", "--sha", "a" * 40, - "--watchdog", "123"]): + + with ( + mock.patch.object(cew, "run", return_value=0) as run_mock, + mock.patch.object(sys, "argv", ["ci_event_wait.py", "--sha", "a" * 40, "--watchdog", "123"]), + ): rc = cew.main() self.assertEqual(rc, 0) self.assertEqual(run_mock.call_args.kwargs.get("watchdog_secs"), 123) @@ -198,34 +206,31 @@ class TestTaskBinding(_IsolatedRoot): def _bind_env(self): import await_intent as ai + importlib.reload(ai) return ai def test_task_bound_operation_resolved(self): ws = self._waitstore() self._new_wait(ws, session_id="s-stale", status="WAITING_EXTERNAL") - bound = self._new_wait(ws, session_id="s-bound", - status="WAITING_EXTERNAL", task_id="T9", - sha="cd" * 20) + bound = self._new_wait(ws, session_id="s-bound", status="WAITING_EXTERNAL", task_id="T9", sha="cd" * 20) ai = self._bind_env() - norm = ai.normalize_action({"kind": "await_operation"}, - task_id="T9") + norm = ai.normalize_action({"kind": "await_operation"}, task_id="T9") self.assertTrue(norm.get("normalized"), norm) self.assertEqual(norm.get("operation_id"), bound["run_ref"]) - self.assertEqual(norm.get("matched_by", norm.get("evidence")), - norm.get("matched_by", norm.get("evidence"))) + self.assertEqual(norm.get("matched_by", norm.get("evidence")), norm.get("matched_by", norm.get("evidence"))) def test_unbound_task_reports_missing(self): ws = self._waitstore() self._new_wait(ws, session_id="s-stale2", status="WAITING_EXTERNAL") ai = self._bind_env() - norm = ai.normalize_action({"kind": "await_operation"}, - task_id="T404") + norm = ai.normalize_action({"kind": "await_operation"}, task_id="T404") self.assertEqual(norm.get("normalized"), False) self.assertIn("T404", norm.get("reason", "")) def _waitstore(self): import waitstore + importlib.reload(waitstore) return waitstore