Compare commits
24
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
efe3871916 | ||
|
|
026a1bd919 | ||
|
|
d9bcebb730 | ||
|
|
2b2639783a | ||
|
|
634821efd4 | ||
|
|
c6c18ae041 | ||
|
|
e03ad9aad9 | ||
|
|
89b603024e | ||
|
|
dac6e97e16 | ||
|
|
96f0e7e263 | ||
|
|
a168cd935e | ||
|
|
a2c4a11b0d | ||
|
|
16164c6f5e | ||
|
|
0722dc2339 | ||
|
|
a7ec1b49d7 | ||
|
|
5b61f8a8bb | ||
|
|
c730719768 | ||
|
|
4fe0f8ee46 | ||
|
|
8999f29041 | ||
|
|
c288f4726d | ||
|
|
308bbb74b7 | ||
|
|
1f01e87453 | ||
|
|
4e8e081f48 | ||
|
|
d01d6eb6d4 |
No files matched your search
@@ -1,6 +1,6 @@
|
||||
# Rune
|
||||
|
||||
**Current release: v0.85.9**
|
||||
**Current release: v0.86.0**
|
||||
|
||||
**Rune is a persistent personal agent designed to accumulate experience, develop reusable competence, and remain recognisably the same agent even when its underlying foundation model changes.**
|
||||
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""completion_file.py — REAL_ASYNC_BRIDGE: durable completion-file delivery
|
||||
with inotify wake. Replaces FIFO+lease for process-wait delivery.
|
||||
|
||||
DESIGN:
|
||||
Producer (bridge/daemon/notify):
|
||||
1. Validates operation identity
|
||||
2. Atomically writes a completion file:
|
||||
resumes/<operation_id>.done
|
||||
containing the event/verdict JSON (write to tmp, rename for atomicity)
|
||||
|
||||
Waiter (ci_event_wait / any continuation consumer):
|
||||
1. Checks if the completion file already exists (event-before-waiter)
|
||||
2. If not: registers inotify watch on the resumes/ directory
|
||||
3. Re-checks after watch registration (closes the registration race)
|
||||
4. Blocks in kernel on inotify — zero CPU, zero timer
|
||||
5. On file-created event: reads the file, verifies authority, returns
|
||||
|
||||
The event is a WAKE SIGNAL. Authority still decides terminal truth.
|
||||
The completion file is durable — survives daemon and waiter restarts.
|
||||
|
||||
WHY NOT FIFO:
|
||||
FIFO is one-shot (token consumed), requires lease management, and
|
||||
fails under session-busy guards. A completion file is atomic, durable,
|
||||
and works across restarts. inotify wakes the process with zero CPU
|
||||
cost and zero latency between file creation and wake.
|
||||
|
||||
WHY NOT POLLING:
|
||||
Polling introduces an arbitrary interval. inotify is a kernel callback
|
||||
that fires exactly when the filesystem changes — the latency tracks
|
||||
the real completion event, not a configured interval.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import struct
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _wr_bridge() -> Path:
|
||||
# WR_ROOT = daemon/infra convention; RUNE_WR_ROOT = repo/scripts
|
||||
# convention. Both honored (WR_ROOT wins) so the daemon, the waiter,
|
||||
# and any isolated test root always resolve the SAME directory.
|
||||
root = os.environ.get("WR_ROOT") or os.environ.get("RUNE_WR_ROOT") or "/mnt/workdrive/wr-bridge"
|
||||
return Path(root)
|
||||
|
||||
|
||||
def _resumes_dir() -> Path:
|
||||
return _wr_bridge() / "resumes"
|
||||
|
||||
|
||||
# inotify constants (Linux)
|
||||
IN_CREATE = 0x100
|
||||
IN_CLOSE_WRITE = 0x8
|
||||
IN_MOVED_TO = 0x80
|
||||
|
||||
|
||||
def completion_path(operation_id: str) -> Path:
|
||||
return _resumes_dir() / f"{operation_id}.done"
|
||||
|
||||
|
||||
def write_completion(operation_id: str, event: dict) -> Path:
|
||||
"""Atomically write a durable completion file. The rename(2) ensures
|
||||
the waiter never sees a partial file."""
|
||||
_resumes_dir().mkdir(parents=True, exist_ok=True)
|
||||
target = completion_path(operation_id)
|
||||
tmp = target.parent / f".{target.name}.tmp.{os.getpid()}"
|
||||
tmp.write_text(json.dumps(event, sort_keys=True) + "\n")
|
||||
os.rename(str(tmp), str(target)) # atomic on same filesystem
|
||||
os.chmod(str(target), 0o666) # cross-user bridge
|
||||
return target
|
||||
|
||||
|
||||
def read_completion(operation_id: str) -> dict | None:
|
||||
"""Read a durable completion file. Returns None if absent."""
|
||||
p = completion_path(operation_id)
|
||||
if not p.is_file():
|
||||
return None
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def completion_exists(operation_id: str) -> bool:
|
||||
return completion_path(operation_id).is_file()
|
||||
|
||||
|
||||
def _inotify_init_and_add_watch(directory: str) -> int:
|
||||
"""Create an inotify instance and add a watch on the directory.
|
||||
Returns the inotify fd."""
|
||||
libc = __import__("ctypes").CDLL(None, use_errno=True)
|
||||
fd = libc.inotify_init()
|
||||
if fd < 0:
|
||||
raise OSError("inotify_init failed")
|
||||
# IN_CREATE | IN_MOVED_TO: catch both direct creation and atomic rename
|
||||
mask = IN_CREATE | IN_MOVED_TO
|
||||
wd = libc.inotify_add_watch(fd, directory.encode(), mask)
|
||||
if wd < 0:
|
||||
os.close(fd)
|
||||
raise OSError(f"inotify_add_watch failed for {directory}")
|
||||
return fd
|
||||
|
||||
|
||||
def _inotify_read(fd: int, timeout_secs: float | None = None) -> list[str]:
|
||||
"""Read pending inotify events. Returns list of filenames that were
|
||||
created/moved-to. Blocks until at least one event or timeout."""
|
||||
poll = select.poll()
|
||||
poll.register(fd, select.POLLIN)
|
||||
events = []
|
||||
buf = os.read(fd, 4096)
|
||||
pos = 0
|
||||
while pos < len(buf):
|
||||
# struct: wd(i32), mask(u32), cookie(u32), name_len(u32)
|
||||
wd, mask, cookie, name_len = struct.unpack_from("iIII", buf, pos)
|
||||
pos += 16
|
||||
name = buf[pos : pos + name_len].rstrip(b"\x00").decode("utf-8", "replace")
|
||||
pos += name_len
|
||||
if mask & (IN_CREATE | IN_MOVED_TO):
|
||||
events.append(name)
|
||||
return events
|
||||
|
||||
|
||||
def wait_for_completion(operation_id: str, *, timeout_secs: float | None = None, poll_callback=None) -> dict | None:
|
||||
"""Block until a completion file exists for this operation.
|
||||
|
||||
Uses inotify for zero-latency, zero-CPU wake. Falls back to
|
||||
polling_callback (if provided) when inotify is unavailable.
|
||||
|
||||
The completion file is a WAKE SIGNAL. The caller must still verify
|
||||
canonical authority after reading it.
|
||||
|
||||
Returns the event dict from the completion file, or None on timeout.
|
||||
"""
|
||||
target = completion_path(operation_id)
|
||||
|
||||
# 1. check-before-watch (event may have arrived before we started)
|
||||
existing = read_completion(operation_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
_resumes_dir().mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
fd = _inotify_init_and_add_watch(str(_resumes_dir()))
|
||||
except OSError:
|
||||
# inotify unavailable: fall back to polling callback
|
||||
if poll_callback:
|
||||
return poll_callback(operation_id, timeout_secs)
|
||||
return None
|
||||
|
||||
try:
|
||||
# 2. re-check after watch registration (closes the race between
|
||||
# the initial check and the watch being active)
|
||||
existing = read_completion(operation_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
# 3. block in kernel on inotify — zero CPU, exact latency
|
||||
import select as _select
|
||||
|
||||
poll = _select.poll()
|
||||
poll.register(fd, _select.POLLIN)
|
||||
deadline = (time.monotonic() + timeout_secs) if timeout_secs else None
|
||||
|
||||
while True:
|
||||
remaining = None
|
||||
if deadline is not None:
|
||||
remaining = max(0.1, deadline - time.monotonic())
|
||||
if remaining <= 0.1:
|
||||
return None # timeout
|
||||
events = poll.poll(remaining * 1000 if remaining else None)
|
||||
if not events:
|
||||
continue # poll timeout, re-check
|
||||
names = _inotify_read(fd)
|
||||
if target.name in names:
|
||||
# 4. file appeared — read it
|
||||
existing = read_completion(operation_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
# partial/corrupt: keep watching
|
||||
finally:
|
||||
os.close(fd)
|
||||
return None
|
||||
|
||||
|
||||
def clear_completion(operation_id: str) -> None:
|
||||
"""Remove a consumed completion file (idempotent cleanup)."""
|
||||
p = completion_path(operation_id)
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
Executable
+799
@@ -0,0 +1,799 @@
|
||||
#!/usr/bin/env python3
|
||||
"""daemon.py — Gitea webhook receiver + reconciliation + resume driver (v0.2).
|
||||
|
||||
W/R-v0.2 invariants implemented here:
|
||||
A. SINGLE-FLIGHT PER SESSION — per-session lease (waitstore.acquire_lease,
|
||||
/proc-validated, atomic) + in-process inflight set. A second READY wait
|
||||
for the same session stays queued until the lease releases.
|
||||
B. BUSY SESSION PROTECTION — before spawning, /proc is scanned for a live
|
||||
`mimo run -s <sid>`-shaped process not owned by our lease. Found → skip
|
||||
launch (wait stays READY). No invented MiMo APIs; the lease is the
|
||||
conservative floor against W/R-created concurrency.
|
||||
C. ATOMIC SESSION LEASE — persisted {session_id, wait_id, pid, pgid,
|
||||
pid_starttime, acquired_at, state}; O_EXCL create, steal only on /proc
|
||||
evidence of a dead owner.
|
||||
D. PROCESS TREE OWNERSHIP — workers spawn with start_new_session=True (own
|
||||
process group). Watch timeout ⇒ SIGTERM group → bounded grace → SIGKILL
|
||||
group → verify zero surviving group members → release lease → truthful
|
||||
READY_TO_RETRY (never terminal-on-timeout).
|
||||
E. TIMEOUT IS NOT SUCCESS — rc==0 maps to verdict terminal
|
||||
(COMPLETED/FAILED); rc!=0 or unverifiable ⇒ RESUME_UNVERIFIED
|
||||
(not schedulable, operator-requeueable); timeout ⇒ READY_TO_RETRY.
|
||||
F. STALE RESUMING RECOVERY — on startup, every RESUMING record is
|
||||
inspected: live lease owner ⇒ adopted and monitored; otherwise ⇒
|
||||
RESUME_UNVERIFIED (+ stale lease released). Never auto-relaunched.
|
||||
G. MULTIPLE WAITS, SAME SESSION — schedulable() orders oldest-first; one
|
||||
continuation finishes before the next starts.
|
||||
H. PIDFILE CORRECTNESS — singleton enforced by flock on daemon.pid; the
|
||||
lock dies with the process, so stale pidfiles (incl. SIGKILL) cannot
|
||||
admit a second daemon.
|
||||
|
||||
The resume command is configurable (WR_RESUME_CMD, JSON list with
|
||||
{session_id}/{msg} placeholders) so the test harness can use a stub; the
|
||||
default is the real mechanism: `mimo run -s <sid> -m <model> <msg>`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import fcntl, json, os, signal, subprocess, sys, threading, time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
import waitstore
|
||||
import verify
|
||||
import completion_file # REAL_ASYNC_BRIDGE R3: durable completion-file delivery
|
||||
|
||||
RESUME_MODEL = "Verboo Code/glm-5.3-flash"
|
||||
RECONCILE_SECS = int(os.environ.get("WR_RECONCILE_SECS", "60"))
|
||||
RECONCILE_EVERY_TICKS = max(1, int(os.environ.get("WR_RECONCILE_TICKS", "3")))
|
||||
WAKE_MIN_AGE_SECS = float(os.environ.get("WR_WAKE_MIN_AGE_SECS", "5"))
|
||||
WAKE_RETRIES = int(os.environ.get("WR_WAKE_RETRIES", "3"))
|
||||
WAKE_RETRY_SECS = float(os.environ.get("WR_WAKE_RETRY_SECS", "4"))
|
||||
DRIVE_LOOP_SECS = int(os.environ.get("WR_DRIVE_SECS", "20"))
|
||||
# REAL_ASYNC_BRIDGE §4: a RESUMING wait older than this grace without
|
||||
# progress is an orphaned resume flow (thread died between lease acquire
|
||||
# and release). Must exceed the longest legitimate in-flight watch
|
||||
# (WATCH_SECS); default 3x. Env-overridable for deterministic tests.
|
||||
WATCH_SECS = int(os.environ.get("WR_WATCH_SECS", "600"))
|
||||
# REAL_ASYNC_BRIDGE §4: a RESUMING wait older than this grace without
|
||||
# progress is an orphaned resume flow (thread died between lease acquire
|
||||
# and release). Must exceed the longest legitimate in-flight watch
|
||||
# (WATCH_SECS); default 3x. Env-overridable for deterministic tests.
|
||||
ORPHAN_GRACE_SECS = int(os.environ.get("WR_ORPHAN_GRACE_SECS", str(3 * WATCH_SECS)))
|
||||
TERM_GRACE_SECS = int(os.environ.get("WR_TERM_GRACE_SECS", "10"))
|
||||
MAX_ATTEMPTS = int(os.environ.get("WR_MAX_ATTEMPTS", "3"))
|
||||
_WR_ROOT = Path(os.environ.get("WR_ROOT", "") or Path(__file__).resolve().parent)
|
||||
PIDFILE = _WR_ROOT / "daemon.pid"
|
||||
LOG = _WR_ROOT / "daemon.log"
|
||||
|
||||
DEFAULT_RESUME_CMD = ["mimo", "run", "-s", "{session_id}", "-m", RESUME_MODEL, "{msg}"]
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
with open(LOG, "a") as f:
|
||||
f.write(f"{time.strftime('%H:%M:%S')} {msg}\n")
|
||||
|
||||
|
||||
def resume_cmd(session_id: str, msg: str, wait_id: str = "", override: list | None = None) -> list[str]:
|
||||
"""T15 §15D: per-wait override wins; placeholders {session_id}/{msg}/
|
||||
{wait_id}. A synthetic controller session (Scaffolding stage driver)
|
||||
is not a MiMo session — the global default cannot resume it, which is
|
||||
exactly why fixed-sleep polling crept back into T12-T14."""
|
||||
if override:
|
||||
return [
|
||||
c.replace("{session_id}", session_id).replace("{msg}", msg).replace("{wait_id}", wait_id) for c in override
|
||||
]
|
||||
tmpl = os.environ.get("WR_RESUME_CMD")
|
||||
cmd = json.loads(tmpl) if tmpl else DEFAULT_RESUME_CMD
|
||||
return [c.replace("{session_id}", session_id).replace("{msg}", msg).replace("{wait_id}", wait_id) for c in cmd]
|
||||
|
||||
|
||||
# ── process-tree evidence & ownership ─────────────────────────────────────
|
||||
|
||||
|
||||
def pgid_members(pgid: int) -> list[int]:
|
||||
out = []
|
||||
for d in Path("/proc").iterdir():
|
||||
if not d.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
stat = (d / "stat").read_text()
|
||||
fields = stat.rsplit(")", 1)[1].split()
|
||||
# zombies: killed but not yet reaped. Counting them makes the
|
||||
# survivor check report the direct child we just KILLed before
|
||||
# communicate() reaps it (observed live 2026-09-12).
|
||||
if fields[0] == "Z":
|
||||
continue
|
||||
if int(fields[2]) == pgid:
|
||||
out.append(int(d.name))
|
||||
except (OSError, IndexError, ValueError):
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def terminate_tree(pgid: int) -> list[int]:
|
||||
"""TERM → bounded grace → KILL. Returns surviving members (must be [])."""
|
||||
try:
|
||||
os.killpg(pgid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
deadline = time.time() + TERM_GRACE_SECS
|
||||
while time.time() < deadline:
|
||||
if not pgid_members(pgid):
|
||||
return []
|
||||
time.sleep(0.2)
|
||||
try:
|
||||
os.killpg(pgid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
for _ in range(50): # up to 10s for KILL to land
|
||||
if not pgid_members(pgid):
|
||||
return []
|
||||
time.sleep(0.2)
|
||||
return pgid_members(pgid)
|
||||
|
||||
|
||||
def stray_worker_pids(session_id: str, exclude_pgids: set[int] = ()) -> list[int]:
|
||||
"""Live `mimo run -s <sid>`-shaped processes not owned by our workers."""
|
||||
out = []
|
||||
for d in Path("/proc").iterdir():
|
||||
if not d.name.isdigit():
|
||||
continue
|
||||
try:
|
||||
parts = [p.decode(errors="replace") for p in (d / "cmdline").read_bytes().split(b"\0") if p]
|
||||
except OSError:
|
||||
continue
|
||||
if not parts:
|
||||
continue
|
||||
joined = " ".join(parts)
|
||||
if session_id not in joined or "-s" not in parts:
|
||||
continue
|
||||
if "mimo" not in joined.lower() and "@mimo-ai" not in joined.lower():
|
||||
continue
|
||||
try:
|
||||
if os.getpgid(int(d.name)) in exclude_pgids:
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
out.append(int(d.name))
|
||||
return out
|
||||
|
||||
|
||||
# ── verification → transition (unchanged authority model) ─────────────────
|
||||
|
||||
|
||||
def _authoritative_verify(rec: dict) -> tuple[str, dict]:
|
||||
"""Authoritative verification with a deterministic test seam.
|
||||
|
||||
Production (WR_VERIFY_STUB unset): real Gitea API via verify.py — the
|
||||
ONLY source of success/failure truth. Test (WR_VERIFY_STUB=path to
|
||||
JSON {"<run_ref-or-sha>": verdict, "default": verdict}): stubbed
|
||||
verdicts so invariant tests never touch the network. The stub is a
|
||||
test fixture, not an alternate authority — no production path sets it.
|
||||
"""
|
||||
stub_path = os.environ.get("WR_VERIFY_STUB", "")
|
||||
if stub_path:
|
||||
try:
|
||||
spec = json.loads(Path(stub_path).read_text())
|
||||
except Exception:
|
||||
return "pending", {"stub_error": "unreadable stub"}
|
||||
key = str(rec.get("run_ref", "") or rec.get("sha", ""))
|
||||
verdict = spec.get(key, spec.get("default", "pending"))
|
||||
return verdict, {"stub": True, "key": key}
|
||||
import verify
|
||||
|
||||
if rec.get("wait_kind") == "RELEASE_DEPLOY":
|
||||
return verify.verify_release_deploy(rec.get("sha", ""), rec.get("repo_root", ""), rec.get("deployed_link", ""))
|
||||
return verify.verify_run(rec.get("run_ref", ""), rec.get("sha", ""), rec.get("expected", "success"))
|
||||
|
||||
|
||||
def try_transition(rec: dict, event: dict | None = None) -> dict:
|
||||
if rec["status"] != "WAITING_EXTERNAL":
|
||||
return rec
|
||||
if rec.get("wait_kind") == "PROCESS_EVENT":
|
||||
# T15 §15D: the matching event IS the authority. The notifier
|
||||
# wrapper observed the real process exit; exit_code decides the
|
||||
# verdict. No Gitea lookup applies. A duplicate event never
|
||||
# reaches here (match applies only to WAITING_EXTERNAL waits).
|
||||
if not event:
|
||||
return rec # no event → nothing authoritative happened
|
||||
try:
|
||||
exit_code = int(event.get("exit_code"))
|
||||
except (TypeError, ValueError):
|
||||
exit_code = None
|
||||
if exit_code is None:
|
||||
return rec # malformed event: stay WAITING (15E hostile case)
|
||||
verdict = "success" if exit_code == 0 else "failure"
|
||||
evidence = {"process_event": True, "exit_code": exit_code, "operation_id": event.get("run_ref", "")}
|
||||
log(f"verify {rec['wait_id']}: {verdict} {json.dumps(evidence)[:120]}")
|
||||
rec["status"] = "READY_TO_RESUME"
|
||||
rec["verdict"] = verdict
|
||||
if verdict == "failure":
|
||||
rec["failure_evidence"] = evidence
|
||||
rec["process_event_evidence"] = evidence
|
||||
waitstore.save(rec)
|
||||
return rec
|
||||
verdict, evidence = _authoritative_verify(rec)
|
||||
log(f"verify {rec['wait_id']}: {verdict} {json.dumps(evidence)[:120]}")
|
||||
if verdict == "success":
|
||||
rec["status"] = "READY_TO_RESUME"
|
||||
rec["verdict"] = "success"
|
||||
elif verdict == "failure":
|
||||
rec["status"] = "READY_TO_RESUME"
|
||||
rec["verdict"] = "failure"
|
||||
rec["failure_evidence"] = verify.failed_job_evidence(rec.get("run_ref", ""))
|
||||
else:
|
||||
return rec # pending/unknown → stay WAITING
|
||||
waitstore.save(rec)
|
||||
return rec
|
||||
|
||||
|
||||
def continuation_msg(rec: dict) -> str:
|
||||
c = rec["continuation"]
|
||||
if rec.get("verdict") == "success":
|
||||
return (
|
||||
f"External wait satisfied (wait {rec['wait_id']}).\n"
|
||||
f"Original objective: {c.get('objective', '')}\n"
|
||||
f"Waited for: {c.get('waited_for', '')}\n"
|
||||
f"Authoritative result: workflow success verified via Gitea API.\n"
|
||||
f"SHA/run: {rec.get('sha', '')}/{rec.get('run_ref', '')}\n"
|
||||
f"Verify: {c.get('verify_when_woken', '')}\n"
|
||||
f"Continue from: {c.get('on_success', '')}"
|
||||
)
|
||||
ev = json.dumps(rec.get("failure_evidence", {}))[:400]
|
||||
return (
|
||||
f"External wait finished with FAILURE (wait {rec['wait_id']}).\n"
|
||||
f"Original objective: {c.get('objective', '')}\n"
|
||||
f"Authoritative result: workflow FAILURE verified via Gitea API.\n"
|
||||
f"Failed job evidence: {ev}\n"
|
||||
f"SHA/run: {rec.get('sha', '')}/{rec.get('run_ref', '')}\n"
|
||||
f"Do NOT treat this as success. Continue from: {c.get('on_failure', '')}"
|
||||
)
|
||||
|
||||
|
||||
# ── resume driver (single-flight, tree-owned) ─────────────────────────────
|
||||
|
||||
|
||||
def resume(rec: dict) -> None:
|
||||
"""Run one continuation to completion. Runs on a dedicated worker thread.
|
||||
Preconditions: caller verified no inflight worker for this session."""
|
||||
sid, wid = rec["session_id"], rec["wait_id"]
|
||||
# REAL_ASYNC_BRIDGE §2 invariant: a successful authority-verified
|
||||
# operation completion must NEVER be overwritten or downgraded by a
|
||||
# legacy session/FIFO delivery failure. Terminal operation state
|
||||
# (COMPLETED/FAILED — stamped by the waiter's own authority
|
||||
# verification) is owned by the OPERATION domain; this function owns
|
||||
# only SESSION-DELIVERY. Re-read from disk: the in-memory copy may be
|
||||
# stale relative to the waiter's terminal stamp.
|
||||
fresh = waitstore.load(wid)
|
||||
if fresh and fresh.get("status") in waitstore.TERMINAL:
|
||||
log(
|
||||
f"resume {wid}: skipped — operation already terminal "
|
||||
f"({fresh['status']}); delivery worker never overrides "
|
||||
f"operation truth"
|
||||
)
|
||||
return
|
||||
rec = fresh or rec
|
||||
msg = continuation_msg(rec)
|
||||
|
||||
# B: busy-session guard before touching the lease
|
||||
strays = stray_worker_pids(sid)
|
||||
if strays:
|
||||
rec["status"] = "READY_TO_RESUME"
|
||||
rec["resume_note"] = f"queued: target session busy (stray pids {strays})"
|
||||
waitstore.save(rec)
|
||||
log(f"queued {wid}: session busy {strays}")
|
||||
return
|
||||
|
||||
# pre-register the lease owner as THIS daemon so acquisition is atomic
|
||||
lease = waitstore.acquire_lease(sid, wid, os.getpid(), os.getpid())
|
||||
if lease is None:
|
||||
live = waitstore.read_lease(sid)
|
||||
rec["status"] = "READY_TO_RESUME"
|
||||
rec["resume_note"] = f"queued: lease held by wait {live and live.get('wait_id')}"
|
||||
waitstore.save(rec)
|
||||
log(f"queued {wid}: lease held")
|
||||
return
|
||||
|
||||
rec["status"] = "RESUMING"
|
||||
rec["resume_attempts"] = rec.get("resume_attempts", 0) + 1
|
||||
waitstore.save(rec)
|
||||
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
resume_cmd(sid, msg, wid, rec.get("resume_cmd")),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
start_new_session=True,
|
||||
)
|
||||
except Exception as e:
|
||||
waitstore.release_lease(sid, wid)
|
||||
rec["status"] = "READY_TO_RETRY" # spawn failed pre-delivery → retryable
|
||||
rec["resume_note"] = f"spawn failed: {e}"
|
||||
waitstore.save(rec)
|
||||
log(f"spawn {wid} failed pre-delivery: {e}")
|
||||
return
|
||||
|
||||
pgid = proc.pid # start_new_session ⇒ pgid == child pid
|
||||
waitstore.update_lease(sid, wait_id=wid, pid=proc.pid, pgid=pgid, pid_starttime=waitstore._pid_starttime(proc.pid))
|
||||
log(f"launched {wid}: pid={proc.pid} pgid={pgid} attempt={rec['resume_attempts']}")
|
||||
|
||||
timed_out = False
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=WATCH_SECS)
|
||||
rc = proc.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
timed_out = True
|
||||
survivors = terminate_tree(pgid)
|
||||
try:
|
||||
stdout, stderr = proc.communicate(timeout=30)
|
||||
except Exception:
|
||||
stdout, stderr = "", ""
|
||||
rc = proc.returncode
|
||||
if survivors:
|
||||
log(f"ERROR {wid}: group {pgid} survivors after KILL: {survivors}")
|
||||
|
||||
waitstore.release_lease(sid, wid)
|
||||
|
||||
if timed_out:
|
||||
# D+E: timeout is NOT success. Tree terminated; retry truthfully.
|
||||
if rec["resume_attempts"] >= MAX_ATTEMPTS:
|
||||
rec["status"] = "RESUME_UNVERIFIED"
|
||||
rec["resume_note"] = (
|
||||
f"watch timeout x{rec['resume_attempts']}; process group {pgid} terminated; attempts exhausted"
|
||||
)
|
||||
else:
|
||||
rec["status"] = "READY_TO_RETRY"
|
||||
rec["resume_note"] = f"watch timeout; process group {pgid} terminated (TERM→KILL, verified); will retry"
|
||||
log(f"timeout {wid}: tree {pgid} terminated, rc={rc}")
|
||||
elif rc == 0:
|
||||
# COMPLETION TRUTH (constraint 4): rc==0 means the headless worker
|
||||
# EXITED CLEANLY — the strongest terminal evidence the supported CLI
|
||||
# provides. Timeout/cancel can never reach this branch. The evidence
|
||||
# chain (rc, output tail, timing) is recorded on the wait record.
|
||||
terminal = "COMPLETED" if rec.get("verdict") == "success" else "FAILED"
|
||||
rec["status"] = terminal
|
||||
rec["resume_evidence"] = {
|
||||
"kind": "worker_exited_rc0",
|
||||
"rc": 0,
|
||||
"output_tail": (stdout or "")[-400:],
|
||||
"finished_at": waitstore.now(),
|
||||
}
|
||||
log(f"resumed {wid}: rc=0 → {terminal}")
|
||||
else:
|
||||
# Nonzero rc: delivery/completion unproven. Honest, not schedulable.
|
||||
rec["status"] = "RESUME_UNVERIFIED"
|
||||
rec["resume_evidence"] = {
|
||||
"kind": "worker_exited_nonzero",
|
||||
"rc": rc,
|
||||
"output_tail": ((stdout or "") + (stderr or ""))[-400:],
|
||||
"finished_at": waitstore.now(),
|
||||
}
|
||||
rec["resume_note"] = f"resume rc={rc}; delivery unverified"
|
||||
log(f"resumed {wid}: rc={rc} → RESUME_UNVERIFIED")
|
||||
waitstore.save(rec)
|
||||
|
||||
|
||||
# ── startup recovery (invariant F) ────────────────────────────────────────
|
||||
|
||||
|
||||
def startup_recovery() -> None:
|
||||
for rec in waitstore.all_waits():
|
||||
if rec.get("status") != "RESUMING":
|
||||
continue
|
||||
sid, wid = rec["session_id"], rec["wait_id"]
|
||||
lease = waitstore.read_lease(sid)
|
||||
if lease and lease.get("wait_id") == wid and waitstore._owner_alive(lease):
|
||||
# Adopt the live worker and monitor it to a bounded conclusion.
|
||||
pgid = lease["pgid"]
|
||||
log(f"recover {wid}: adopted live worker pgid={pgid}")
|
||||
deadline = time.time() + WATCH_SECS
|
||||
while time.time() < deadline and pgid_members(pgid):
|
||||
time.sleep(2)
|
||||
survivors = terminate_tree(pgid) if pgid_members(pgid) else []
|
||||
waitstore.release_lease(sid, wid)
|
||||
rec["status"] = "RESUME_UNVERIFIED"
|
||||
rec["resume_note"] = "adopted post-restart; outcome unverifiable" + (
|
||||
f"; survivors killed: {survivors}" if survivors else ""
|
||||
)
|
||||
waitstore.save(rec)
|
||||
log(f"recover {wid}: adopted worker ended → RESUME_UNVERIFIED")
|
||||
else:
|
||||
if lease:
|
||||
waitstore.release_lease(sid, wid) # stale
|
||||
rec["status"] = "RESUME_UNVERIFIED"
|
||||
rec["resume_note"] = "recovered at daemon startup: no live owner; delivery unverifiable"
|
||||
waitstore.save(rec)
|
||||
log(f"recover {wid}: no live owner → RESUME_UNVERIFIED")
|
||||
|
||||
|
||||
# ── reconciler + drive loop (invariant G: deterministic queue) ────────────
|
||||
|
||||
_inflight: dict[str, threading.Thread] = {}
|
||||
_inflight_lock = threading.Lock()
|
||||
|
||||
|
||||
def drive_ready() -> None:
|
||||
for rec in waitstore.schedulable():
|
||||
sid = rec["session_id"]
|
||||
# CONTENTION POLICY (constraint 2): MiMo exposes NO supported
|
||||
# busy/idle session signal (verified 2026-09-12: `mimo session list
|
||||
# --format json` has no status field; no `session status` command;
|
||||
# no status column in the session schema; no documented ACP/serve
|
||||
# endpoint). Therefore automatic injection into interactive sessions
|
||||
# is NOT YET SAFE and stays disabled. Only waits explicitly declared
|
||||
# target_kind="headless" (disposable/owned sessions) auto-dispatch.
|
||||
if rec.get("target_kind") != "headless":
|
||||
if not rec.get("hold_reason"):
|
||||
rec["hold_reason"] = (
|
||||
"INTERACTIVE_AUTO_RESUME = NOT YET SAFE: "
|
||||
"no supported MiMo busy/idle signal; "
|
||||
"queued until operator releases"
|
||||
)
|
||||
waitstore.save(rec)
|
||||
continue
|
||||
if rec.get("ext", {}).get("resume_mode") == "self":
|
||||
# REAL_ASYNC_BRIDGE §3: self-resuming wait (completion-file /
|
||||
# inotify waiter owns its continuation). The daemon has NO
|
||||
# delivery role here — dispatching a delivery worker would
|
||||
# stamp a phantom rc!=0 over the waiter's own truthful
|
||||
# terminal state (the canonical-state contradiction).
|
||||
# NB: keyed on the explicit self marker, NOT on resume_cmd
|
||||
# absence — records without inline resume_cmd may still be
|
||||
# dispatchable via the WR_RESUME_CMD env template.
|
||||
continue
|
||||
with _inflight_lock:
|
||||
t = _inflight.get(sid)
|
||||
if t is not None and t.is_alive():
|
||||
continue # single-flight: session already being driven
|
||||
# B: never launch into a busy session
|
||||
strays = stray_worker_pids(sid)
|
||||
if strays:
|
||||
log(f"drive: {rec['wait_id']} held, session busy {strays}")
|
||||
continue
|
||||
th = threading.Thread(target=resume, args=(rec,), daemon=True, name=f"resume-{rec['wait_id']}")
|
||||
_inflight[sid] = th
|
||||
# start INSIDE the lock: a just-created thread is not yet
|
||||
# is_alive() until start() runs — starting outside the lock let
|
||||
# the same drive cycle dispatch a second worker for the session
|
||||
# (single-flight race, caught by the T2 invariant test).
|
||||
th.start()
|
||||
log(f"drive: dispatched {rec['wait_id']} (session {sid})")
|
||||
|
||||
|
||||
def reconcile_once() -> None:
|
||||
for rec in waitstore.pending():
|
||||
if rec["status"] != "WAITING_EXTERNAL":
|
||||
continue
|
||||
created = rec.get("created_at", "")
|
||||
try:
|
||||
import calendar
|
||||
|
||||
age = time.time() - calendar.timegm(time.strptime(created, "%Y-%m-%dT%H:%M:%SZ"))
|
||||
except ValueError:
|
||||
continue
|
||||
if age > RECONCILE_SECS:
|
||||
try_transition(rec)
|
||||
|
||||
|
||||
_tick = 0
|
||||
|
||||
|
||||
def orphan_sweep() -> int:
|
||||
"""REAL_ASYNC_BRIDGE §4: permanent mid-run orphan-lease recovery.
|
||||
|
||||
FAILURE DETECTION ONLY — never involved in normal completion
|
||||
scheduling. A wait stuck in RESUMING past ORPHAN_GRACE_SECS means the
|
||||
resume flow died between lease acquire and release (worker thread
|
||||
lost, exception mid-watch). Reconcile exactly as startup_recovery
|
||||
would have: release the lease, stamp RESUME_UNVERIFIED truthfully
|
||||
(delivery unverifiable), never auto-relaunch.
|
||||
"""
|
||||
import calendar as _cal
|
||||
|
||||
healed = 0
|
||||
for rec in waitstore.all_waits():
|
||||
if rec.get("status") != "RESUMING":
|
||||
continue
|
||||
try:
|
||||
updated = _cal.timegm(time.strptime(rec.get("last_updated", ""), "%Y-%m-%dT%H:%M:%SZ"))
|
||||
except ValueError:
|
||||
continue
|
||||
if time.time() - updated < ORPHAN_GRACE_SECS:
|
||||
continue # legitimate in-flight watch (up to WATCH_SECS)
|
||||
wid, sid = rec["wait_id"], rec["session_id"]
|
||||
lease = waitstore.read_lease(sid)
|
||||
owner_dead = lease is not None and not waitstore._owner_alive(lease)
|
||||
daemon_owned = lease is not None and lease.get("pid") == os.getpid()
|
||||
if lease is None or owner_dead or daemon_owned:
|
||||
if lease is not None:
|
||||
waitstore.release_lease(sid, wid)
|
||||
if rec.get("resume_attempts", 0) >= MAX_ATTEMPTS:
|
||||
rec["status"] = "RESUME_UNVERIFIED"
|
||||
else:
|
||||
rec["status"] = "READY_TO_RETRY"
|
||||
rec["resume_note"] = (
|
||||
"orphaned mid-run: resume flow died between lease acquire "
|
||||
"and delivery ("
|
||||
+ ("lease owner dead" if owner_dead else "resume thread lost" if daemon_owned else "no lease")
|
||||
+ "); orphan sweep recovered"
|
||||
)
|
||||
waitstore.save(rec)
|
||||
log(f"orphan-sweep {wid}: {rec['status']} — {rec['resume_note']}")
|
||||
healed += 1
|
||||
return healed
|
||||
|
||||
|
||||
def reconciler() -> None:
|
||||
global _tick
|
||||
while True:
|
||||
# v0.3: the local-runner wake is the fast path; periodic
|
||||
# authoritative verification demotes to a slower fallback
|
||||
# (missed-event + daemon-restart recovery), not API polling.
|
||||
_tick += 1
|
||||
if _tick % RECONCILE_EVERY_TICKS == 0:
|
||||
try:
|
||||
reconcile_once()
|
||||
except Exception as e:
|
||||
log(f"reconcile error: {e}")
|
||||
try:
|
||||
orphan_sweep() # §4 failure detection — bounded, periodic
|
||||
except Exception as e:
|
||||
log(f"orphan-sweep error: {e}")
|
||||
try:
|
||||
drive_ready()
|
||||
except Exception as e:
|
||||
log(f"drive error: {e}")
|
||||
with _inflight_lock:
|
||||
for sid, t in list(_inflight.items()):
|
||||
if not t.is_alive():
|
||||
del _inflight[sid]
|
||||
time.sleep(DRIVE_LOOP_SECS)
|
||||
|
||||
|
||||
# ── webhook receiver (wake signal only) ───────────────────────────────────
|
||||
|
||||
|
||||
def wake_sweep(event: dict, eid: str) -> dict:
|
||||
"""Sweep WAITING_EXTERNAL waits through authoritative verification.
|
||||
|
||||
Runs on a background thread (the hook's HTTP response returns
|
||||
instantly, minimising the runner's offline window). If any verified
|
||||
wait got 'pending' — the normal case for a job that finished
|
||||
microseconds ago: the post-task hook fires after results are reported
|
||||
but before the runner's final task acknowledgement flips server-side
|
||||
status — retry on a BOUNDED event-triggered schedule (default 4s x3),
|
||||
never a polling loop. The final ack lands milliseconds after the hook
|
||||
exits, so the first retry almost always sees the terminal state.
|
||||
"""
|
||||
arrival = time.time()
|
||||
log(f"local wake {eid}: {json.dumps(event)[:140]}")
|
||||
import calendar
|
||||
|
||||
def sweep() -> tuple[int, int, int]:
|
||||
verified = advanced = pending = 0
|
||||
for rec in waitstore.pending():
|
||||
if rec["status"] != "WAITING_EXTERNAL":
|
||||
continue
|
||||
try:
|
||||
age = time.time() - calendar.timegm(time.strptime(rec.get("created_at", ""), "%Y-%m-%dT%H:%M:%SZ"))
|
||||
except ValueError:
|
||||
continue
|
||||
if age < WAKE_MIN_AGE_SECS:
|
||||
continue
|
||||
verified += 1
|
||||
before = rec["status"]
|
||||
try_transition(rec)
|
||||
if rec["status"] != before and rec["status"] == "READY_TO_RESUME":
|
||||
advanced += 1
|
||||
elif rec["status"] == "WAITING_EXTERNAL":
|
||||
pending += 1
|
||||
return verified, advanced, pending
|
||||
|
||||
verified, advanced, pending = sweep()
|
||||
log(
|
||||
f"local wake {eid}: verified={verified} advanced={advanced} "
|
||||
f"pending={pending} sweep_ms={int((time.time() - arrival) * 1000)}"
|
||||
)
|
||||
for attempt in range(1, WAKE_RETRIES + 1):
|
||||
# Retry while the sweep didn't conclusively advance anything:
|
||||
# pending verdicts (ack-window) AND fresh-wait skips (age gate).
|
||||
# Bounded, event-triggered — not a polling loop.
|
||||
if not pending and verified:
|
||||
break
|
||||
time.sleep(WAKE_RETRY_SECS)
|
||||
verified, advanced, pending = sweep()
|
||||
log(
|
||||
f"local wake {eid}: retry{attempt} verified={verified} "
|
||||
f"advanced={advanced} pending={pending} "
|
||||
f"total_ms={int((time.time() - arrival) * 1000)}"
|
||||
)
|
||||
return {"event_id": eid}
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
if self.path == "/process-event":
|
||||
# T15 §15D: process-exit events from builder/controller
|
||||
# notifiers. Minimal dedicated endpoint — the Gitea webhook
|
||||
# path stays untouched. Payload: {operation_id, exit_code,
|
||||
# completed_at} + OPTIONAL provider/event_type/sha (T16
|
||||
# §16C/§16E: the CI adapter registers as provider="gitea",
|
||||
# event_type="ci_completed"; defaults preserve the T15 builder
|
||||
# identity so existing notifiers are unaffected). The event is
|
||||
# a WAKE, never truth — resumed controllers re-read canonical
|
||||
# authority before any verdict.
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length)
|
||||
try:
|
||||
payload = json.loads(body) if body else {}
|
||||
except json.JSONDecodeError:
|
||||
self.send_response(400)
|
||||
self.end_headers()
|
||||
return
|
||||
event = {
|
||||
"provider": str(payload.get("provider", "builder")),
|
||||
"event_type": str(payload.get("event_type", "builder_exit")),
|
||||
"run_ref": str(payload.get("operation_id", "")) or None,
|
||||
"sha": payload.get("sha"),
|
||||
"exit_code": payload.get("exit_code"),
|
||||
"completed_at": payload.get("completed_at") or "",
|
||||
}
|
||||
event = {k: v for k, v in event.items() if v is not None}
|
||||
eid = waitstore.record_event(event)
|
||||
log(f"process-event {eid}: {json.dumps(event)[:140]}")
|
||||
# REAL_ASYNC_BRIDGE (R3): durable completion-file delivery.
|
||||
# Every event reaching the bridge is written as
|
||||
# resumes/<operation_id>.done so inotify waiters wake on the
|
||||
# kernel event (R4) with no FIFO/lease machinery, and the
|
||||
# event survives daemon AND waiter restarts (R9). Atomic
|
||||
# rename; duplicate events rewrite identical content (R10).
|
||||
_op_id = str(payload.get("operation_id", ""))
|
||||
if _op_id:
|
||||
try:
|
||||
# Stable identity only: duplicate deliveries of the
|
||||
# same logical event must produce byte-identical
|
||||
# completion files (R10 idempotence by construction).
|
||||
# completed_at is volatile and lives in the durable
|
||||
# event log; the file is a WAKE SIGNAL, never truth.
|
||||
_stable = {
|
||||
k: event.get(k)
|
||||
for k in ("provider", "event_type", "run_ref", "sha", "exit_code")
|
||||
if event.get(k) is not None
|
||||
}
|
||||
completion_file.write_completion(_op_id, _stable)
|
||||
except Exception as e: # noqa: BLE001 — delivery best-effort; wait-record transition + watchdog recovery still cover the lost file
|
||||
log(f"process-event {eid}: completion-file write failed: {e}")
|
||||
matched = 0
|
||||
for rec in waitstore.pending():
|
||||
if rec["status"] == "WAITING_EXTERNAL" and waitstore.match(rec, event):
|
||||
matched += 1
|
||||
try_transition(rec, event)
|
||||
# matched READY controller waits are dispatched IMMEDIATELY
|
||||
# (event-driven, no floor). Guard: only never-resumed waits —
|
||||
# a duplicate/late event must not double-dispatch (15E);
|
||||
# single-flight lease + this guard together make resume
|
||||
# idempotent.
|
||||
for rec in waitstore.pending():
|
||||
if rec["status"] == "READY_TO_RESUME" and rec.get("resume_cmd") and rec.get("resume_attempts", 0) == 0:
|
||||
threading.Thread(target=resume, args=(rec,), daemon=True, name=f"resume-{rec['wait_id']}").start()
|
||||
matched += 1
|
||||
try:
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"event_id": eid, "matched": matched}).encode())
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass
|
||||
return
|
||||
if self.path == "/wake":
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length)
|
||||
try:
|
||||
payload = json.loads(body) if body else {}
|
||||
except json.JSONDecodeError:
|
||||
self.send_response(400)
|
||||
self.end_headers()
|
||||
return
|
||||
event = {
|
||||
"provider": "gitea-local-runner",
|
||||
"task_id": str(payload.get("task_id", "")) or None,
|
||||
"run_id": str(payload.get("run_id", "")) or None,
|
||||
"repository": str(payload.get("repository", "")) or None,
|
||||
"job_result": str(payload.get("job_result", "")) or None,
|
||||
}
|
||||
event = {k: v for k, v in event.items() if v}
|
||||
eid = waitstore.record_event(event)
|
||||
# Respond instantly; sweep on a worker thread. The hook is
|
||||
# fire-and-forget (curl -m 2) — a broken pipe on write is
|
||||
# harmless and must not propagate.
|
||||
try:
|
||||
threading.Thread(
|
||||
target=wake_sweep, args=(event, eid), daemon=True, name=f"wake-{payload.get('task_id', '?')}"
|
||||
).start()
|
||||
except Exception as e:
|
||||
log(f"wake spawn error: {e}")
|
||||
try:
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"queued": True, "event_id": eid}).encode())
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass
|
||||
return
|
||||
if self.path != "/webhook":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length)
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
self.send_response(400)
|
||||
self.end_headers()
|
||||
return
|
||||
action = payload.get("action") or payload.get("event_type") or ""
|
||||
wf = payload.get("workflow_run") or {}
|
||||
event = {
|
||||
"provider": "gitea",
|
||||
"event_type": f"workflow.{action}" if wf else action,
|
||||
"sha": (wf.get("head_sha") or payload.get("sha") or "")[:12] or None,
|
||||
"run_ref": str(wf.get("id") or payload.get("run_number") or "") or None,
|
||||
}
|
||||
event = {k: v for k, v in event.items() if v}
|
||||
eid = waitstore.record_event(event)
|
||||
log(f"webhook event {eid}: {json.dumps(event)[:140]}")
|
||||
matched = 0
|
||||
for rec in waitstore.pending():
|
||||
if rec["status"] == "WAITING_EXTERNAL" and waitstore.match(rec, event):
|
||||
matched += 1
|
||||
try_transition(rec)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"event_id": eid, "matched": matched}).encode())
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/health":
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{"ok":true}')
|
||||
return
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
|
||||
def main() -> int:
|
||||
port = int(sys.argv[1]) if len(sys.argv) > 1 else 8791
|
||||
|
||||
# H (constraint 1): flock is the SINGLETON AUTHORITY. daemon.pid's
|
||||
# content is diagnostic metadata only — it never gates startup, never
|
||||
# implies liveness, and is overwritten after the exclusive lock is held.
|
||||
# Stale pid content (incl. SIGKILL debris) is thus harmless by design.
|
||||
PIDFILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock_fd = os.open(PIDFILE, os.O_CREAT | os.O_RDWR, 0o644)
|
||||
try:
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
log("refusing start: another daemon holds the pidfile flock")
|
||||
print("daemon already running (pidfile flock held)", file=sys.stderr)
|
||||
return 1
|
||||
os.ftruncate(lock_fd, 0)
|
||||
os.write(lock_fd, f"{os.getpid()}\n".encode()) # diagnostic, lock held for life
|
||||
|
||||
startup_recovery()
|
||||
t = threading.Thread(target=reconciler, daemon=True)
|
||||
t.start()
|
||||
log(f"daemon starting on :{port}")
|
||||
srv = ThreadingHTTPServer(("0.0.0.0", port), Handler)
|
||||
try:
|
||||
srv.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+246
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
"""verify.py — authoritative Gitea verification (webhook ≠ success).
|
||||
|
||||
Queries the Gitea API for the TRUE state of the run/job/commit the wait
|
||||
references. Returns (verdict, evidence). verdict ∈ {success, failure,
|
||||
pending, unknown}. Only this module may move a wait out of
|
||||
WAITING_EXTERNAL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import json, subprocess
|
||||
from pathlib import Path
|
||||
|
||||
HOST = "https://gitea.thingscouldgetdicey.online"
|
||||
REPO = "conductor/rune"
|
||||
|
||||
|
||||
def _token() -> str:
|
||||
# Canonical ladder without importing Rune: env → token file → git store
|
||||
import os
|
||||
|
||||
tok = os.environ.get("GITEA_TOKEN", "").strip()
|
||||
if tok:
|
||||
return tok
|
||||
tf = os.path.expanduser("~/.config/opencode/secrets/gitea-token")
|
||||
if os.path.isfile(tf):
|
||||
return open(tf).read().strip()
|
||||
r = subprocess.run(
|
||||
["git", "credential", "fill"],
|
||||
input="protocol=https\nhost=gitea.thingscouldgetdicey.online\nusername=conductor\n\n",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
for line in r.stdout.splitlines():
|
||||
if line.startswith("password="):
|
||||
return line[9:]
|
||||
raise RuntimeError("no gitea credential available")
|
||||
|
||||
|
||||
def _api(path: str) -> tuple[int, object]:
|
||||
import urllib.request
|
||||
|
||||
req = urllib.request.Request(f"{HOST}/api/v1/repos/{REPO}{path}", headers={"Authorization": f"token {_token()}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as r:
|
||||
return r.status, json.loads(r.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, {"error": e.read().decode()[:200]}
|
||||
|
||||
|
||||
def _jobs_list(run_ref: str):
|
||||
"""GET jobs for a run, unwrapping the {'jobs': [...]} envelope."""
|
||||
s, data = _api(f"/actions/runs/{run_ref}/jobs")
|
||||
if s != 200:
|
||||
return s, None
|
||||
if isinstance(data, dict):
|
||||
data = data.get("jobs", [])
|
||||
return s, data if isinstance(data, list) else None
|
||||
|
||||
|
||||
def _resolve_run_id(run_ref: str, sha: str) -> str:
|
||||
"""Task-list ids are NOT run ids. If run_ref lookup 404s, resolve the
|
||||
real run id by head SHA via /actions/runs (T15: run 6907 vs task 15743)."""
|
||||
if run_ref:
|
||||
s, jobs = _jobs_list(run_ref)
|
||||
if s == 200 and jobs:
|
||||
return run_ref
|
||||
if sha:
|
||||
s, runs = _api(f"/actions/runs?limit=30")
|
||||
items = runs.get("workflow_runs", []) if isinstance(runs, dict) else (runs if isinstance(runs, list) else [])
|
||||
# One SHA fans out to multiple runs (lint + deploy-only). Prefer a
|
||||
# substantive (non-skipped) run; a skipped-only run proves nothing.
|
||||
fallback = ""
|
||||
for r in items:
|
||||
if str(r.get("head_sha", "")).startswith(sha):
|
||||
rid = str(r.get("id"))
|
||||
if r.get("conclusion") != "skipped":
|
||||
return rid
|
||||
fallback = fallback or rid
|
||||
return fallback
|
||||
return ""
|
||||
|
||||
|
||||
def verify_run(run_ref: str, sha: str, expected: str) -> tuple[str, dict]:
|
||||
"""run_ref = actions run id (or task id — resolved via SHA lookup).
|
||||
|
||||
Defect fixed 2026-09-12 (surfaced by W/R-v0.3 live proof, run 7110):
|
||||
the jobs query here used raw _api + isinstance(jobs, list), but this
|
||||
Gitea wraps jobs in the {"jobs": [...]} envelope — the check always
|
||||
failed, so EVERY EXACT_WORKFLOW verdict silently fell through to the
|
||||
aggregate commit-status fallback, where unrelated runs on the same
|
||||
SHA (multi-run fan-out: ci.yml + deploy-only) poison the verdict.
|
||||
Use _jobs_list (unwraps the envelope), same as _resolve_run_id."""
|
||||
run_ref = _resolve_run_id(run_ref, sha)
|
||||
if run_ref:
|
||||
s, jobs = _jobs_list(run_ref)
|
||||
if s == 200 and isinstance(jobs, list) and jobs:
|
||||
conclusions = {j.get("name"): j.get("conclusion") for j in jobs}
|
||||
statuses = {j.get("status") for j in jobs}
|
||||
if statuses <= {"completed"}:
|
||||
verdict = "success" if set(conclusions.values()) <= {"success", "skipped"} else "failure"
|
||||
return verdict, {"run_ref": run_ref, "jobs": conclusions}
|
||||
return "pending", {"run_ref": run_ref, "statuses": sorted(statuses)}
|
||||
if sha:
|
||||
s, st = _api(f"/commits/{sha}/status")
|
||||
if s == 200 and isinstance(st, dict):
|
||||
state = st.get("state", "unknown") # success|error|failure|pending|warning
|
||||
rid = _resolve_run_id("", sha) # attach run id so failure evidence works
|
||||
if state == "success":
|
||||
return "success", {"sha": sha, "state": state, "run_ref": rid}
|
||||
if state in ("error", "failure"):
|
||||
return "failure", {"sha": sha, "state": state, "run_ref": rid}
|
||||
return "pending", {"sha": sha, "state": state}
|
||||
return "unknown", {"run_ref": run_ref, "sha": sha}
|
||||
|
||||
|
||||
def failed_job_evidence(run_ref: str, sha: str = "") -> dict:
|
||||
"""For failure continuations: exact failing job + conclusion."""
|
||||
run_ref = _resolve_run_id(run_ref, sha)
|
||||
if not run_ref:
|
||||
return {}
|
||||
s, jobs = _jobs_list(run_ref)
|
||||
if s != 200 or not isinstance(jobs, list):
|
||||
return {}
|
||||
bad = [j for j in jobs if j.get("conclusion") not in ("success", "skipped", None)]
|
||||
return {
|
||||
"failed_jobs": [
|
||||
{"name": j.get("name"), "conclusion": j.get("conclusion"), "url": j.get("url", "")} for j in bad
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# ── RELEASE_DEPLOY verification (W/R v0.1, overnight handoff Part 2) ──────
|
||||
# Aggregate commit status is NOT a valid completion authority for Rune's
|
||||
# release/deploy flow: the release successor SHA, not the source SHA, is what
|
||||
# converges (observed live: wr-1789179026 stranded on pending while 0.66.15
|
||||
# was deployed). The authority chain is:
|
||||
# source SHA → source runs terminal → release successor commit (descendant)
|
||||
# → successor runs success → deployed/current identity converges.
|
||||
|
||||
|
||||
def _git(args: list[str], repo_root: str) -> str:
|
||||
import subprocess
|
||||
|
||||
r = subprocess.run(["git"] + args, capture_output=True, text=True, cwd=repo_root or None, timeout=15)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError(f"git {' '.join(args[:2])}: {r.stderr.strip()[:120]}")
|
||||
return r.stdout
|
||||
|
||||
|
||||
def _runs_for_sha(sha: str) -> list[dict]:
|
||||
s, runs = _api("/actions/runs?limit=50")
|
||||
items = runs.get("workflow_runs", []) if isinstance(runs, dict) else []
|
||||
return [r for r in items if str(r.get("head_sha", "")).startswith(sha)]
|
||||
|
||||
|
||||
def _sha_verdict_from_runs(runs: list[dict]) -> tuple[str, dict]:
|
||||
"""success | failure | pending from a SHA's workflow runs.
|
||||
Terminal verdict ONLY when every run is completed — mixed states
|
||||
(some success, some still in flight) must stay PENDING (observed live:
|
||||
a SHA with 1 completed-success + 1 queued + 1 in-progress was judged
|
||||
success mid-flight; wr-1789182522 2026-09-12)."""
|
||||
if not runs:
|
||||
return "pending", {"note": "no runs found"}
|
||||
non_terminal = [r for r in runs if r.get("status") != "completed"]
|
||||
if non_terminal:
|
||||
return "pending", {"in_flight": len(non_terminal)}
|
||||
conclusions = {r.get("name"): r.get("conclusion") for r in runs}
|
||||
bad = {n: c for n, c in conclusions.items() if c == "failure"}
|
||||
if bad:
|
||||
return "failure", {"failed": bad}
|
||||
return "success", {"runs": conclusions}
|
||||
|
||||
|
||||
def verify_release_deploy(source_sha: str, repo_root: str = "", deployed_link: str = "") -> tuple[str, dict]:
|
||||
"""RELEASE_DEPLOY authority chain. Returns (verdict, evidence)."""
|
||||
source_sha = (source_sha or "")[:12]
|
||||
ev: dict = {"source_sha": source_sha, "kind": "RELEASE_DEPLOY"}
|
||||
|
||||
# 1. Source SHA's own runs must be terminal and healthy.
|
||||
src_runs = _runs_for_sha(source_sha)
|
||||
src_verdict, src_ev = _sha_verdict_from_runs(src_runs)
|
||||
ev["source"] = src_ev
|
||||
if src_verdict != "success":
|
||||
return src_verdict, ev
|
||||
|
||||
# 2. Release successor: first chore(release) commit descending from source.
|
||||
if not repo_root:
|
||||
return "pending", {**ev, "note": "no repo_root — cannot check lineage"}
|
||||
try:
|
||||
out = _git(
|
||||
[
|
||||
"log",
|
||||
"--format=%H %s",
|
||||
"--reverse",
|
||||
f"{source_sha}..master",
|
||||
"--grep=^chore\\(release\\)",
|
||||
"--extended-regexp",
|
||||
],
|
||||
repo_root,
|
||||
)
|
||||
except RuntimeError as e:
|
||||
return "pending", {**ev, "note": str(e)}
|
||||
successors = [l for l in out.splitlines() if l.strip()]
|
||||
if not successors:
|
||||
return "pending", {**ev, "note": "no release successor yet"}
|
||||
succ_line = successors[0]
|
||||
succ_sha_full, _, succ_subject = succ_line.partition(" ")
|
||||
succ_sha = succ_sha_full[:12]
|
||||
ev["successor"] = {"sha": succ_sha, "subject": succ_subject[:80]}
|
||||
|
||||
# Ancestry is guaranteed by the range, but verify explicitly (deterministic
|
||||
# lineage over title guessing):
|
||||
try:
|
||||
_git(["merge-base", "--is-ancestor", source_sha, succ_sha_full], repo_root)
|
||||
except RuntimeError:
|
||||
return "failure", {**ev, "note": "successor is not a descendant of source"}
|
||||
|
||||
# 3. Successor runs must be terminal and healthy.
|
||||
succ_runs = _runs_for_sha(succ_sha)
|
||||
succ_verdict, succ_ev = _sha_verdict_from_runs(succ_runs)
|
||||
ev["successor"]["runs"] = succ_ev
|
||||
if succ_verdict != "success":
|
||||
return succ_verdict, ev
|
||||
|
||||
# 4. Deployed identity convergence. A mismatch is FAILURE only when the
|
||||
# successor's runs are ALL completed — if any is still running/queued the
|
||||
# deploy may simply not have propagated yet → PENDING (observed live:
|
||||
# reconciler checked mid-deploy, verdict failure, deployment converged
|
||||
# ~2 min later; wr-1789182522 2026-09-12).
|
||||
if deployed_link:
|
||||
link = Path(deployed_link)
|
||||
if link.is_symlink() or link.is_dir():
|
||||
deployed = str(link.resolve())
|
||||
deployed_id = Path(deployed).name
|
||||
ev["deployed"] = deployed_id
|
||||
if not deployed_id.startswith(succ_sha[:12]):
|
||||
still = [r for r in _runs_for_sha(succ_sha) if r.get("status") != "completed"]
|
||||
if still:
|
||||
return "pending", {**ev, "note": "deploy propagating (runs in flight)"}
|
||||
return "failure", {**ev, "note": f"deployed {deployed_id} != successor {succ_sha} (all runs terminal)"}
|
||||
else:
|
||||
return "failure", {**ev, "note": f"deployed link missing: {deployed_link}"}
|
||||
|
||||
return "success", ev
|
||||
Executable
+337
@@ -0,0 +1,337 @@
|
||||
#!/usr/bin/env python3
|
||||
"""waitstore.py — wait/resume state machine v0.2 (developer tooling, NOT Rune).
|
||||
|
||||
States:
|
||||
RUNNABLE → WAITING_EXTERNAL → READY_TO_RESUME → RESUMING → COMPLETED | FAILED
|
||||
↑ │
|
||||
│ ├─(watch timeout, tree killed)
|
||||
│ └─(rc!=0 / unverifiable)
|
||||
READY_TO_RETRY ←─────┘
|
||||
RESUME_UNVERIFIED : delivery after crash/timeout cannot be proven.
|
||||
TERMINAL-ish — never auto-scheduled; explicit
|
||||
`requeue()` required to make it READY_TO_RESUME.
|
||||
TIMED_OUT : external event never arrived within the wait budget.
|
||||
|
||||
Waits persist as waits/<wait_id>.json (atomic tmp+rename).
|
||||
Leases persist as leases/<session_id>.lease.json (atomic tmp+rename,
|
||||
validated against /proc so a dead or PID-reused owner is stealable).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import fcntl, json, os, tempfile, time, uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(os.environ.get("WR_ROOT", "") or Path(__file__).resolve().parent)
|
||||
WAITS = ROOT / "waits"
|
||||
EVENTS = ROOT / "events"
|
||||
LEASES = ROOT / "leases"
|
||||
|
||||
SCHEDULABLE = {"READY_TO_RESUME", "READY_TO_RETRY"}
|
||||
VALID_STATES = {
|
||||
"RUNNABLE",
|
||||
"WAITING_EXTERNAL",
|
||||
"READY_TO_RESUME",
|
||||
"RESUMING",
|
||||
"COMPLETED",
|
||||
"FAILED",
|
||||
"READY_TO_RETRY",
|
||||
"RESUME_UNVERIFIED",
|
||||
"TIMED_OUT",
|
||||
}
|
||||
TERMINAL = {"COMPLETED", "FAILED", "RESUME_UNVERIFIED", "TIMED_OUT"}
|
||||
|
||||
|
||||
def _atomic_write(path: Path, data: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=".tmp-")
|
||||
try:
|
||||
with os.fdopen(fd, "w") as f:
|
||||
json.dump(data, f, indent=1, sort_keys=True)
|
||||
os.replace(tmp, path)
|
||||
# T17 17C: the bridge root is SHARED across users (deploy waiter
|
||||
# runs as the runner user; daemon as nick). mkstemp creates 0600 —
|
||||
# daemon re-saves locked the runner user out of records it must
|
||||
# read (waitstore.load in await_resume_event; find_matching_event
|
||||
# replay scan silently skipped them). 0644: owner writes, all read.
|
||||
os.chmod(path, 0o644)
|
||||
except BaseException:
|
||||
if os.path.exists(tmp):
|
||||
os.unlink(tmp)
|
||||
raise
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
|
||||
def new_wait(
|
||||
*,
|
||||
session_id: str,
|
||||
provider: str,
|
||||
event_type: str,
|
||||
repo: str,
|
||||
sha: str = "",
|
||||
run_ref: str = "",
|
||||
expected: str = "success",
|
||||
wait_kind: str = "EXACT_WORKFLOW",
|
||||
target_kind: str = "interactive",
|
||||
repo_root: str = "",
|
||||
deployed_link: str = "",
|
||||
objective: str = "",
|
||||
waited_for: str = "",
|
||||
verify_when_woken: str = "",
|
||||
on_success: str = "",
|
||||
on_failure: str = "",
|
||||
resume_cmd: list | None = None,
|
||||
) -> dict:
|
||||
wid = f"wr-{int(time.time())}-{uuid.uuid4().hex[:6]}"
|
||||
rec = {
|
||||
"wait_id": wid,
|
||||
"session_id": session_id,
|
||||
"created_at": now(),
|
||||
"provider": provider,
|
||||
"event_type": event_type,
|
||||
"repo": repo,
|
||||
"sha": sha,
|
||||
"run_ref": run_ref,
|
||||
"expected": expected,
|
||||
# T15 §15D: PROCESS_EVENT — a wait whose authoritative verdict is
|
||||
# the matching process-exit event itself (exit_code carried on the
|
||||
# event), e.g. a builder/controller process completing. The event
|
||||
# IS the authority here (the notifier wrapper observed the real
|
||||
# process exit); no Gitea lookup applies.
|
||||
"wait_kind": wait_kind
|
||||
if wait_kind in ("EXACT_WORKFLOW", "RELEASE_DEPLOY", "PROCESS_EVENT")
|
||||
else "EXACT_WORKFLOW",
|
||||
# CONTENTION POLICY: "interactive" (default, conservative) never
|
||||
# auto-dispatches — MiMo exposes NO supported busy/idle session
|
||||
# signal, so injecting into a possibly-busy interactive session is
|
||||
# NOT YET SAFE. Only explicitly declared headless/disposable
|
||||
# targets ("headless") are auto-resumed.
|
||||
"target_kind": target_kind if target_kind in ("headless", "interactive") else "interactive",
|
||||
"repo_root": repo_root,
|
||||
"deployed_link": deployed_link,
|
||||
# T15 §15D: per-wait resume command override (placeholder template,
|
||||
# same substitution as WR_RESUME_CMD: {session_id}/{msg}/{wait_id}).
|
||||
# A synthetic controller session (e.g. a Scaffolding stage driver)
|
||||
# is NOT a MiMo session — the global mimo-shaped default cannot
|
||||
# resume it, which is exactly why fixed-sleep polling crept back
|
||||
# in T12-T14. Per-wait override makes ANY registered continuation
|
||||
# target resumable through the same bridge.
|
||||
"resume_cmd": list(resume_cmd) if resume_cmd else None,
|
||||
"status": "WAITING_EXTERNAL",
|
||||
"last_verified": "",
|
||||
"resume_attempts": 0,
|
||||
"continuation": {
|
||||
"objective": objective,
|
||||
"waited_for": waited_for,
|
||||
"verify_when_woken": verify_when_woken,
|
||||
"on_success": on_success,
|
||||
"on_failure": on_failure,
|
||||
},
|
||||
}
|
||||
_atomic_write(WAITS / f"{wid}.json", rec)
|
||||
return rec
|
||||
|
||||
|
||||
def load(wid: str) -> dict | None:
|
||||
p = WAITS / f"{wid}.json"
|
||||
return json.loads(p.read_text()) if p.is_file() else None
|
||||
|
||||
|
||||
def save(rec: dict) -> None:
|
||||
assert rec.get("status") in VALID_STATES, rec.get("status")
|
||||
rec["last_updated"] = now()
|
||||
_atomic_write(WAITS / f"{rec['wait_id']}.json", rec)
|
||||
|
||||
|
||||
def all_waits() -> list[dict]:
|
||||
out = []
|
||||
for p in WAITS.glob("*.json"):
|
||||
try:
|
||||
out.append(json.loads(p.read_text()))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def pending() -> list[dict]:
|
||||
return [r for r in all_waits() if r.get("status") in {"WAITING_EXTERNAL"} | SCHEDULABLE]
|
||||
|
||||
|
||||
def schedulable() -> list[dict]:
|
||||
"""READY waits, oldest first — deterministic queue order."""
|
||||
rs = [r for r in all_waits() if r.get("status") in SCHEDULABLE]
|
||||
rs.sort(key=lambda r: (r.get("created_at", ""), r.get("wait_id", "")))
|
||||
return rs
|
||||
|
||||
|
||||
def requeue(wid: str, note: str) -> dict | None:
|
||||
"""Operator-explicit requeue of a RESUME_UNVERIFIED/RESUME_FAILED wait."""
|
||||
rec = load(wid)
|
||||
if not rec:
|
||||
return None
|
||||
rec["status"] = "READY_TO_RESUME"
|
||||
rec["resume_note"] = note
|
||||
save(rec)
|
||||
return rec
|
||||
|
||||
|
||||
# ── events (wake signals, idempotent) ─────────────────────────────────────
|
||||
|
||||
|
||||
def record_event(event: dict) -> str:
|
||||
import hashlib
|
||||
|
||||
body = json.dumps(event, sort_keys=True)
|
||||
eid = hashlib.sha256(body.encode()).hexdigest()[:16]
|
||||
f = EVENTS / f"{eid}.json"
|
||||
if not f.is_file():
|
||||
_atomic_write(f, {"event_id": eid, "received_at": now(), "event": event})
|
||||
return eid
|
||||
|
||||
|
||||
def match(rec: dict, event: dict) -> bool:
|
||||
if rec.get("provider") != event.get("provider"):
|
||||
return False
|
||||
if rec.get("event_type") and event.get("event_type") and rec["event_type"] != event["event_type"]:
|
||||
return False
|
||||
if rec.get("sha") and event.get("sha") and rec["sha"] != event["sha"]:
|
||||
return False
|
||||
if rec.get("run_ref") and event.get("run_ref") and rec["run_ref"] != event["run_ref"]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ── leases (per-session single-flight, /proc-validated) ───────────────────
|
||||
#
|
||||
# Invariant 3 (PID reuse): liveness is NEVER "PID exists". The lease records
|
||||
# pid + pgid + process starttime (/proc/<pid>/stat field 22); all three must
|
||||
# match or the owner is a different process wearing a recycled PID.
|
||||
#
|
||||
# Invariant 1 (flock as authority): the lease file's flock serializes
|
||||
# read-modify-write across processes and dies with its holder, so a crashed
|
||||
# daemon can never wedge the lock. The lease JSON + /proc decide who OWNS the
|
||||
# session; the flock only prevents racing acquirers from double-stealing.
|
||||
|
||||
_lease_fds: dict[str, int] = {} # session_id → open fd holding the flock
|
||||
|
||||
|
||||
def _pid_starttime(pid: int) -> str | None:
|
||||
"""Field 22 of /proc/<pid>/stat — distinguishes PID reuse."""
|
||||
try:
|
||||
stat = Path(f"/proc/{pid}/stat").read_text()
|
||||
return stat.rsplit(")", 1)[1].split()[19]
|
||||
except (OSError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _owner_alive(lease: dict) -> bool:
|
||||
pid = lease.get("pid")
|
||||
if not pid or not Path(f"/proc/{pid}").is_dir():
|
||||
return False
|
||||
if _pid_starttime(pid) != lease.get("pid_starttime"):
|
||||
return False # PID reused by an unrelated process
|
||||
try:
|
||||
if os.getpgid(pid) != lease.get("pgid"):
|
||||
return False
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def lease_path(session_id: str) -> Path:
|
||||
# T16 §16C finding (observed live in daemon.log): session ids like
|
||||
# "controller-<op>-Verboo Code/glm-5.3-flash" contain "/" — the raw
|
||||
# embed made lease creation raise FileNotFoundError, so resume
|
||||
# delivery for those waits could never succeed. Sanitize to a flat
|
||||
# filesystem-safe name; the hash suffix prevents collisions between
|
||||
# ids that sanitize identically.
|
||||
import hashlib as _hashlib
|
||||
import re as _re
|
||||
|
||||
safe = _re.sub(r"[^A-Za-z0-9._-]+", "_", session_id)[:80]
|
||||
digest = _hashlib.sha256(session_id.encode()).hexdigest()[:10]
|
||||
return LEASES / f"{safe}-{digest}.lease.json"
|
||||
|
||||
|
||||
def read_lease(session_id: str) -> dict | None:
|
||||
p = lease_path(session_id)
|
||||
if not p.is_file():
|
||||
return None
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _locked_lease_fd(session_id: str) -> int | None:
|
||||
"""Open the lease file and take its flock (non-blocking). The fd stays
|
||||
open (lock held) until release_lease closes it."""
|
||||
if session_id in _lease_fds:
|
||||
return _lease_fds[session_id] # already hold it
|
||||
LEASES.mkdir(parents=True, exist_ok=True)
|
||||
fd = os.open(lease_path(session_id), os.O_CREAT | os.O_RDWR, 0o600)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
os.close(fd)
|
||||
return None # another process holds the lock — mid-acquire
|
||||
_lease_fds[session_id] = fd
|
||||
return fd
|
||||
|
||||
|
||||
def _write_lease_locked(session_id: str, lease: dict) -> None:
|
||||
fd = _lease_fds[session_id]
|
||||
os.ftruncate(fd, 0)
|
||||
os.lseek(fd, 0, os.SEEK_SET)
|
||||
os.write(fd, json.dumps(lease, indent=1, sort_keys=True).encode())
|
||||
os.fsync(fd)
|
||||
|
||||
|
||||
def acquire_lease(session_id: str, wait_id: str, pid: int, pgid: int) -> dict | None:
|
||||
"""Atomically create or steal the session lease. Returns the lease dict,
|
||||
or None if a live owner holds it (or another process is mid-steal).
|
||||
Staleness is decided by /proc identity evidence, never by age alone."""
|
||||
fd = _locked_lease_fd(session_id)
|
||||
if fd is None:
|
||||
return None
|
||||
existing = read_lease(session_id)
|
||||
if existing and _owner_alive(existing):
|
||||
_lease_fds.pop(session_id, None)
|
||||
os.close(fd)
|
||||
return None # live worker owns this session
|
||||
lease = {
|
||||
"session_id": session_id,
|
||||
"wait_id": wait_id,
|
||||
"pid": pid,
|
||||
"pgid": pgid,
|
||||
"pid_starttime": _pid_starttime(pid),
|
||||
"acquired_at": now(),
|
||||
"state": "RESUMING",
|
||||
}
|
||||
_write_lease_locked(session_id, lease)
|
||||
return lease
|
||||
|
||||
|
||||
def update_lease(session_id: str, **fields) -> None:
|
||||
lease = read_lease(session_id) or {}
|
||||
lease.update(fields)
|
||||
_write_lease_locked(session_id, lease)
|
||||
|
||||
|
||||
def release_lease(session_id: str, wait_id: str) -> bool:
|
||||
"""Release only if the current lease belongs to this wait (no stealing
|
||||
a successor's lease during a slow shutdown path). Closes the flock fd."""
|
||||
lease = read_lease(session_id)
|
||||
if not (lease and lease.get("wait_id") == wait_id):
|
||||
return False
|
||||
fd = _lease_fds.pop(session_id, None)
|
||||
if fd is not None:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
except OSError:
|
||||
pass
|
||||
os.close(fd)
|
||||
lease_path(session_id).unlink(missing_ok=True)
|
||||
return True
|
||||
@@ -1,54 +1,92 @@
|
||||
import { tool } from "@mimo-ai/plugin"
|
||||
import { execSync } from "child_process"
|
||||
|
||||
// await_operation — T21.3 §21.3C: FIRST-CLASS async wait tool.
|
||||
// Uses the plugin tool() API with args (not parameters).
|
||||
|
||||
export const AwaitOperation = tool({
|
||||
description: [
|
||||
"Wait for an asynchronous external operation (CI, deploy, release) to complete.",
|
||||
"Registers on the event bridge, yields until the completion event arrives, then returns the verified result.",
|
||||
"USE THIS instead of `sleep N; curl status` — it is the canonical event-driven wait.",
|
||||
].join(" "),
|
||||
export default tool({
|
||||
description: "Wait for an external operation (CI, deploy, release) to complete. Call this when you cannot make progress until an external process finishes. You do NOT need to know the sha, op_type, or duration — Rune's controller resolves those from the bound PendingOperation. Just call this when you want to wait.",
|
||||
args: {
|
||||
sha: tool.schema.string().describe(
|
||||
"The exact commit SHA the operation is running against"),
|
||||
deadline_secs: tool.schema.number().optional().describe(
|
||||
"Maximum wait in seconds (default 900)"),
|
||||
operation_id: tool.schema.string().optional().describe("Optional: the operation to wait for. If omitted, the controller resolves the current pending operation automatically."),
|
||||
task_id: tool.schema.string().optional().describe("Optional: the active task id (e.g. T4). When provided, resolution is scoped to operations bound to this task — stale global operations are never consulted."),
|
||||
sha: tool.schema.string().optional().describe("OPTIONAL HINT — non-authoritative. The controller uses this only to help resolve which operation you mean."),
|
||||
op_type: tool.schema.string().optional().describe("OPTIONAL HINT — non-authoritative. The controller derives the canonical operation type from repository/Gitea state."),
|
||||
deadline_secs: tool.schema.number().optional().describe("OPTIONAL HINT — non-authoritative. The controller applies governed policy timing. Real user deadlines must be stated in conversation."),
|
||||
},
|
||||
async execute(args, ctx) {
|
||||
const repoRoot = "/mnt/workdrive/issue161-harness"
|
||||
const deadline = args.deadline_secs || 900
|
||||
const cmd = [
|
||||
"python3", `${repoRoot}/scripts/ci_event_wait.py`,
|
||||
"--sha", args.sha,
|
||||
"--deadline", String(deadline),
|
||||
].join(" ")
|
||||
async execute(args) {
|
||||
// INTENT NORMALIZATION — model output is intent; the controller
|
||||
// owns execution semantics. All model args are hints recorded for
|
||||
// observability; canonical truth comes from the bound operation.
|
||||
const parts = []
|
||||
if (args.operation_id) parts.push(`--operation-id ${args.operation_id}`)
|
||||
if (args.task_id) parts.push(`--task-id ${args.task_id}`)
|
||||
if (args.sha) parts.push(`--sha ${args.sha}`)
|
||||
if (args.op_type) parts.push(`--model-op-type ${args.op_type}`)
|
||||
if (args.deadline_secs) parts.push(`--model-deadline ${args.deadline_secs}`)
|
||||
const normCmd = `python3 /mnt/workdrive/issue161-harness/scripts/await_intent_cli.py normalize ${parts.join(" ")}`
|
||||
let norm: any
|
||||
try {
|
||||
const normOut = execSync(normCmd, {
|
||||
cwd: "/mnt/workdrive/issue161-harness",
|
||||
encoding: "utf-8" as const,
|
||||
timeout: 120000,
|
||||
env: { ...process.env, RUNE_WR_ROOT: "/mnt/workdrive/wr-bridge" },
|
||||
})
|
||||
norm = JSON.parse(normOut.trim().split("\n").find((l: string) => l.startsWith("{")) || "{}")
|
||||
} catch (e: any) {
|
||||
return JSON.stringify({ ok: false, stage: "normalization", error: (e.stderr || e.message || "?").slice(0, 300) })
|
||||
}
|
||||
if (norm.normalized === "AMBIGUOUS") {
|
||||
return JSON.stringify({ ok: false, stage: "normalization", ...norm })
|
||||
}
|
||||
if (!norm.normalized) {
|
||||
return JSON.stringify({ ok: false, stage: "normalization", ...norm })
|
||||
}
|
||||
// canonical continuation: stored/derived op_type + POLICY deadline
|
||||
// (the sha for the wait comes from the normalization resolution,
|
||||
// NOT from the model's args)
|
||||
const waitSha = norm.sha_for_wait || args.sha || ""
|
||||
if (!waitSha) {
|
||||
return JSON.stringify({ ok: false, stage: "resolution", error: "no sha resolved for the pending operation", norm })
|
||||
}
|
||||
const cmd = `python3 /mnt/workdrive/issue161-harness/scripts/ci_event_wait.py --sha ${waitSha} --deadline ${norm.deadline_secs} --op-type ${norm.op_type}`
|
||||
try {
|
||||
const result = execSync(cmd, {
|
||||
cwd: repoRoot,
|
||||
timeout: (deadline + 60) * 1000,
|
||||
cwd: "/mnt/workdrive/issue161-harness",
|
||||
timeout: (norm.deadline_secs + 60) * 1000,
|
||||
encoding: "utf-8" as const,
|
||||
env: {
|
||||
...process.env,
|
||||
DEPLOY_SHA: args.sha,
|
||||
RUNE_WR_ROOT: "/mnt/workdrive/wr-bridge",
|
||||
RUNE_WR_CODE: "/home/nick/.local/share/mimo-wait-resume",
|
||||
env: { ...process.env, DEPLOY_SHA: waitSha, RUNE_WR_ROOT: "/mnt/workdrive/wr-bridge", RUNE_WR_CODE: "/home/nick/.local/share/mimo-wait-resume" },
|
||||
})
|
||||
const jl = result.trim().split("\n").find((l: string) => l.startsWith("{"))
|
||||
const envelope = jl ? JSON.parse(jl) : { ok: false, error: "no output" }
|
||||
return JSON.stringify({
|
||||
...envelope,
|
||||
normalization: {
|
||||
model_deadline_ignored: norm.model_deadline_ignored,
|
||||
model_op_type_ignored: norm.model_op_type_ignored,
|
||||
model_sha_ignored: !!args.sha && args.sha !== waitSha,
|
||||
canonical_op_type: norm.op_type,
|
||||
canonical_authority: norm.authority,
|
||||
deadline_provenance: norm.deadline_provenance,
|
||||
matched_by: norm.matched_by || norm.evidence || "auto-resolved",
|
||||
},
|
||||
})
|
||||
const lines = result.trim().split("\n")
|
||||
const jsonLine = lines.find((l: string) => l.startsWith("{"))
|
||||
if (jsonLine) return jsonLine
|
||||
return "await_operation: no JSON output"
|
||||
} catch (e: any) {
|
||||
const stdout = e.stdout || ""
|
||||
const lines = stdout.trim().split("\n")
|
||||
const jsonLine = lines.find((l: string) => l.startsWith("{"))
|
||||
if (jsonLine) return jsonLine
|
||||
return JSON.stringify({
|
||||
ok: false,
|
||||
error: `await_operation failed: ${(e.stderr || e.message || "?").slice(0, 200)}`,
|
||||
})
|
||||
const lines = (e.stdout || "").trim().split("\n")
|
||||
const jl = lines.find((l: string) => l.startsWith("{"))
|
||||
if (jl) {
|
||||
const envelope = JSON.parse(jl)
|
||||
return JSON.stringify({
|
||||
...envelope,
|
||||
normalization: {
|
||||
model_deadline_ignored: norm.model_deadline_ignored,
|
||||
model_op_type_ignored: norm.model_op_type_ignored,
|
||||
model_sha_ignored: !!args.sha && args.sha !== waitSha,
|
||||
canonical_op_type: norm.op_type,
|
||||
canonical_authority: norm.authority,
|
||||
deadline_provenance: norm.deadline_provenance,
|
||||
matched_by: norm.matched_by || norm.evidence || "auto-resolved",
|
||||
},
|
||||
})
|
||||
}
|
||||
return JSON.stringify({ ok: false, stage: "wait", error: (e.stderr || e.message || "?").slice(0, 300) })
|
||||
}
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,167 @@
|
||||
|
||||
// bash — executes a shell command. PROCESS-OWNERSHIP CONTRACT (v2):
|
||||
//
|
||||
// MODE A (tool-owned): the child is launched in its own process group;
|
||||
// stdout/stderr are CAPTURED pipes (never the TUI tty). On timeout or
|
||||
// failure the ENTIRE process group is signalled (SIGTERM, grace,
|
||||
// SIGKILL) so grandchildren cannot survive the invocation, then the
|
||||
// result is finalized. After finalization NO descendant retains a
|
||||
// writer into MiMo's streams.
|
||||
//
|
||||
// MODE B (explicitly detached): a command suffixed with " &" that the
|
||||
// user/model intends to outlive the tool (nohup/setsid/disown-style)
|
||||
// is executed via the detached helper: stdin=/dev/null, stdout/stderr
|
||||
// -> durable log file, own session (setsid), no MiMo PTY inheritance.
|
||||
// The tool returns the log path as the durable handle.
|
||||
//
|
||||
// Wait-intent compilation (sleep/check/poll aimed at a pending operation)
|
||||
// is preserved: interception happens at the SEMANTIC level via
|
||||
// await_intent route-bash, and a NO-OP for anything that is not a genuine
|
||||
// continuation of a live pending operation. Arbitrary shell syntax
|
||||
// (quotes, pipes, semicolons) passes through untouched — the router
|
||||
// response is only trusted when it carries normalized=true AND
|
||||
// ContinuePendingOperation with a concrete operation bound.
|
||||
import { tool } from "@mimo-ai/plugin"
|
||||
import { spawn, execFile } from "child_process"
|
||||
|
||||
const REPO = "/mnt/workdrive/issue161-harness"
|
||||
const WR_ROOT = "/mnt/workdrive/wr-bridge"
|
||||
|
||||
// ── MODE A: process-group-owned capture execution ──────────────────────
|
||||
interface OwnedResult { stdout: string; stderr: string; status: number; timedOut: boolean }
|
||||
|
||||
function runOwned(command: string, timeoutMs: number): Promise<OwnedResult> {
|
||||
return new Promise((resolve) => {
|
||||
// /bin/sh -c "command" in its OWN process group (detached:true makes
|
||||
// the child a session/group leader we can signal as a WHOLE).
|
||||
const child = spawn("/bin/sh", ["-c", command], {
|
||||
cwd: REPO,
|
||||
stdio: ["ignore", "pipe", "pipe"], // NEVER inherit the TUI tty
|
||||
env: process.env,
|
||||
detached: true, // own process group
|
||||
})
|
||||
let stdout = "", stderr = ""
|
||||
let timedOut = false
|
||||
let settled = false
|
||||
const outChunks: Buffer[] = []
|
||||
const errChunks: Buffer[] = []
|
||||
child.stdout!.on("data", (d) => outChunks.push(d))
|
||||
child.stderr!.on("data", (d) => errChunks.push(d))
|
||||
|
||||
const killGroup = (sig: NodeJS.Signals) => {
|
||||
try { process.kill(-child.pid!, sig) } catch { /* already gone */ }
|
||||
}
|
||||
const finalize = (status: number) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
clearTimeout(timer)
|
||||
// detach listeners, then resolve — no late listeners remain
|
||||
child.stdout!.removeAllListeners()
|
||||
child.stderr!.removeAllListeners()
|
||||
resolve({ stdout: Buffer.concat(outChunks).toString(), stderr: Buffer.concat(errChunks).toString(), status, timedOut })
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true
|
||||
killGroup("SIGTERM")
|
||||
// bounded grace for clean shutdown, then SIGKILL the group
|
||||
setTimeout(() => killGroup("SIGKILL"), 2000)
|
||||
// the exit/close event settles; a hard stop-guard settles anyway
|
||||
setTimeout(() => finalize(124), 4000)
|
||||
}, timeoutMs)
|
||||
|
||||
child.on("error", (e) => finalize(-1))
|
||||
child.on("close", (code) => finalize(code ?? -1))
|
||||
})
|
||||
}
|
||||
|
||||
// ── MODE B: explicit detachment for "cmd &" invocations ────────────────
|
||||
function runDetached(command: string): Promise<string> {
|
||||
return new Promise((resolve) => {
|
||||
const logPath = `/tmp/mimo-detached-${Date.now()}-${Math.floor(Math.random() * 1e6)}.log`
|
||||
const out = `exec >>"${logPath}" 2>&1; exec 0</dev/null; ${command}`
|
||||
const child = spawn("/bin/sh", ["-c", `setsid /bin/sh -c '${out.replace(/'/g, `'\\''`)}'`], {
|
||||
cwd: REPO,
|
||||
stdio: "ignore", // child rebinds its own fds — no PTY inheritance
|
||||
env: process.env,
|
||||
detached: true,
|
||||
// unref: MiMo's event loop does not wait for it
|
||||
})
|
||||
child.unref()
|
||||
resolve(JSON.stringify({
|
||||
detached: true,
|
||||
log: logPath,
|
||||
note: "process detached (setsid, own fds); output streams to the log file only; fd 0/1/2 do NOT reference the MiMo tty",
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
// ── wait-intent routing (semantic level, preserved from v1) ────────────
|
||||
async function tryCompileWaitIntent(command: string): Promise<string | null> {
|
||||
let routeOut: string
|
||||
try {
|
||||
routeOut = await new Promise<string>((resolve, reject) => {
|
||||
execFile("python3", [`${REPO}/scripts/await_intent_cli.py`, "route-bash", "--command", command], {
|
||||
cwd: REPO, timeout: 60000, encoding: "utf-8",
|
||||
env: { ...process.env, RUNE_WR_ROOT: WR_ROOT },
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
}, (err, stdout) => { if (err && !stdout) reject(err); else resolve(stdout || "") })
|
||||
})
|
||||
} catch {
|
||||
return null // router unavailable: normal execution (never block on routing)
|
||||
}
|
||||
const jl = routeOut.trim().split("\n").find((l) => l.startsWith("{"))
|
||||
if (!jl) return null
|
||||
let norm: any
|
||||
try { norm = JSON.parse(jl) } catch { return null }
|
||||
// STRICT gate: compile ONLY a normalized, concrete continuation.
|
||||
if (!(norm.normalized === true &&
|
||||
norm.action === "ContinuePendingOperation" &&
|
||||
norm.operation_id && norm.sha_for_wait)) return null
|
||||
const waitCmd = `python3 ${REPO}/scripts/ci_event_wait.py --sha ${norm.sha_for_wait} --deadline ${norm.deadline_secs} --op-type ${norm.op_type}`
|
||||
const r = await runOwned(waitCmd, (norm.deadline_secs + 60) * 1000)
|
||||
const jline = r.stdout.trim().split("\n").find((l) => l.startsWith("{"))
|
||||
if (jline) {
|
||||
return JSON.stringify({
|
||||
...JSON.parse(jline),
|
||||
intent_compilation: {
|
||||
raw_command_kind: norm.raw_command_kind || "sleep_poll",
|
||||
literal_sleep_executed: false,
|
||||
operation_id: norm.operation_id,
|
||||
canonical_op_type: norm.op_type,
|
||||
authority: norm.authority,
|
||||
},
|
||||
})
|
||||
}
|
||||
return JSON.stringify({ ok: false, stage: "compiled-wait", error: (r.stderr || "no output").slice(0, 300) })
|
||||
}
|
||||
|
||||
export default tool({
|
||||
description: "Run a shell command. If the command expresses external wait intent (sleep/check/poll aimed at CI, deploy, or release operations with pending operations), it is automatically compiled to Rune's event-driven continuation — you do not need to manage wait mechanics. Plain local commands execute normally. Commands ending in ' &' run detached with output logged to a file.",
|
||||
args: {
|
||||
command: tool.schema.string().describe("Shell command to run"),
|
||||
},
|
||||
async execute(args) {
|
||||
// 1. wait-intent compilation (semantic, strict-gated)
|
||||
try {
|
||||
const compiled = await tryCompileWaitIntent(args.command)
|
||||
if (compiled !== null) return compiled
|
||||
} catch { /* fall through to normal execution */ }
|
||||
|
||||
// 2. MODE B: explicit detachment for backgrounded invocations
|
||||
const cmd = args.command
|
||||
if (/\b(nohup|setsid)\b/.test(cmd) || /&\s*$/.test(cmd)) {
|
||||
return await runDetached(cmd)
|
||||
}
|
||||
|
||||
// 3. MODE A: tool-owned, group-killed, captured execution
|
||||
const r = await runOwned(cmd, 120000)
|
||||
const text = r.stdout + (r.stderr ? `\n[stderr]\n${r.stderr}` : "")
|
||||
if (r.timedOut) {
|
||||
return `exit timed-out (process group terminated): ${text.slice(0, 500)}`
|
||||
}
|
||||
if (r.status !== 0) {
|
||||
return `exit ${r.status}: ${text.slice(0, 500) || "?"}`
|
||||
}
|
||||
return text || "(no output)"
|
||||
},
|
||||
})
|
||||
+1
-1
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "rune"
|
||||
version = "0.85.9"
|
||||
version = "0.86.0"
|
||||
description = "Rune — a stateful coding assistant with persistent memory, orchestration, and TUI"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile pyproject.toml -o /tmp/req3.txt --extra dev --extra memory --python-version 3.11
|
||||
# uv pip compile pyproject.toml -o /tmp/req_new.txt --extra dev --extra memory --python-version 3.11
|
||||
aiohappyeyeballs==2.7.1
|
||||
# via aiohttp
|
||||
aiohttp==3.14.3
|
||||
@@ -64,7 +64,7 @@ nodeenv==1.10.0
|
||||
# via pyright
|
||||
packaging==26.3
|
||||
# via pytest
|
||||
platformdirs==4.11.10
|
||||
platformdirs==4.11.11
|
||||
# via textual
|
||||
pluggy==1.6.0
|
||||
# via
|
||||
|
||||
+77
-1
@@ -205,11 +205,16 @@ def derive_op_type(sha: str) -> tuple[str, str]:
|
||||
def normalize_action(action: dict, *,
|
||||
model_deadline: Any = None,
|
||||
user_deadline: Any = None,
|
||||
contract_deadline: Any = None) -> dict:
|
||||
contract_deadline: Any = None,
|
||||
task_id: str = "") -> dict:
|
||||
"""Normalize a model continuation action into the canonical continuation.
|
||||
|
||||
action: {kind: await_operation|sleep_poll|check_status|unknown,
|
||||
sha?: str, operation_id?: str, op_type?: str, raw?: str}
|
||||
task_id: REAL_ASYNC_BRIDGE §9 — when the caller knows its task
|
||||
binding, pending-operation resolution is scoped to operations
|
||||
registered with that task_id; stale global operations are
|
||||
never consulted when a binding exists.
|
||||
Returns:
|
||||
{normalized: bool, action: "ContinuePendingOperation", operation_id,
|
||||
op_type, authority, deadline_secs, deadline_provenance,
|
||||
@@ -222,11 +227,61 @@ def normalize_action(action: dict, *,
|
||||
model_deadline_ignored = model_deadline is not None
|
||||
model_op_type_ignored = model_op_type is not None
|
||||
|
||||
# 0. task binding scope (§9): task-bound operations beat global state.
|
||||
# Stored operation wins over ANY model hint (sha/op_type/deadline).
|
||||
if task_id and not op_id:
|
||||
bound = [r for r in pending_operations()
|
||||
if str(r.get("ext", {}).get("task_id", "")) == task_id]
|
||||
if len(bound) == 1:
|
||||
rec = bound[0]
|
||||
norm = _canonical_from_record(rec, model_deadline_ignored,
|
||||
model_op_type_ignored,
|
||||
"task_bound",
|
||||
user_deadline, contract_deadline)
|
||||
norm["sha_for_wait"] = _rec_sha(rec)
|
||||
norm["task_id"] = task_id
|
||||
return norm
|
||||
if len(bound) > 1:
|
||||
return {"normalized": "AMBIGUOUS",
|
||||
"candidates": [_rec_operation_id(c) for c in bound],
|
||||
"reason": f"multiple pending operations bound to task {task_id}"}
|
||||
return {"normalized": False,
|
||||
"reason": f"no pending operation bound to task {task_id}"}
|
||||
|
||||
# 1. explicit operation_id wins
|
||||
if op_id:
|
||||
from rune import event_wait as ew # noqa: E402
|
||||
rec = ew.operation_wait(op_id)
|
||||
if rec is None:
|
||||
# REAL_ASYNC_BRIDGE §10: a FRESH operation has no wait record
|
||||
# yet — registration happens at the park step (ci_event_wait).
|
||||
# For an explicit, well-formed, EXISTING commit id, return the
|
||||
# provisional canonical continuation (policy deadline, own-runs
|
||||
# authority) so the model-facing tool can park on first call.
|
||||
# A typo'd/ghost sha is refused here (A-patch lesson): the
|
||||
# commit must exist in the repository.
|
||||
import re as _re
|
||||
m = _re.fullmatch(r"ci-([0-9a-f]{40})", op_id)
|
||||
if m:
|
||||
sha40 = m.group(1)
|
||||
probe = subprocess.run(
|
||||
["git", "-C", str(REPO), "cat-file", "-e",
|
||||
f"{sha40}^{{commit}}"],
|
||||
capture_output=True, timeout=15)
|
||||
if probe.returncode == 0:
|
||||
return {
|
||||
"normalized": True,
|
||||
"action": "ContinuePendingOperation",
|
||||
"operation_id": op_id,
|
||||
"sha_for_wait": sha40,
|
||||
"op_type": "branch_ci",
|
||||
"authority": "ci_runs",
|
||||
"deadline_secs": POLICY_DEADLINES["branch_ci"],
|
||||
"deadline_provenance": "GOVERNED_OPERATION_POLICY",
|
||||
"model_deadline_ignored": model_deadline_ignored,
|
||||
"model_op_type_ignored": model_op_type_ignored,
|
||||
"evidence": "explicit_id_fresh_registration",
|
||||
}
|
||||
return {"normalized": False,
|
||||
"reason": f"operation_id {op_id} has no wait record"}
|
||||
return _canonical_from_record(rec, model_deadline_ignored,
|
||||
@@ -239,6 +294,27 @@ def normalize_action(action: dict, *,
|
||||
# mistyped model sha must never create a ghost operation (the A-patch
|
||||
# live failure: a typo'd sha derived branch_ci for a nonexistent commit
|
||||
# and parked a wait on a ghost).
|
||||
# 2. no explicit operation_id: resolve by sha / pending operations.
|
||||
# H-PATCH: when the model provides NO sha (operation-id-first mode),
|
||||
# resolve from unambiguous pending operations — the model should not
|
||||
# need to reconstruct a 40-char sha to express "wait for it".
|
||||
if not sha:
|
||||
all_pending = pending_operations()
|
||||
if len(all_pending) == 1:
|
||||
rec = all_pending[0]
|
||||
norm = _canonical_from_record(rec, model_deadline_ignored,
|
||||
model_op_type_ignored,
|
||||
"auto_resolved_no_sha",
|
||||
user_deadline, contract_deadline)
|
||||
norm["sha_for_wait"] = _rec_sha(rec)
|
||||
return norm
|
||||
if len(all_pending) > 1:
|
||||
return {"normalized": "AMBIGUOUS",
|
||||
"candidates": [_rec_operation_id(c) for c in all_pending],
|
||||
"reason": "multiple pending operations, no sha hint"}
|
||||
return {"normalized": False,
|
||||
"reason": "no pending operations and no sha provided"}
|
||||
|
||||
candidates = pending_operations(sha=sha)
|
||||
if len(candidates) > 1 and not sha:
|
||||
return {"normalized": "AMBIGUOUS",
|
||||
|
||||
@@ -21,6 +21,9 @@ def main() -> int:
|
||||
n = sub.add_parser("normalize", help="normalize a continuation intent")
|
||||
n.add_argument("--sha", default="")
|
||||
n.add_argument("--operation-id", default="")
|
||||
n.add_argument("--task-id", default="",
|
||||
help="§9 task binding: scope pending resolution to "
|
||||
"operations registered for this task")
|
||||
n.add_argument("--model-op-type", default=None)
|
||||
n.add_argument("--model-deadline", default=None)
|
||||
n.add_argument("--user-deadline", default=None,
|
||||
@@ -60,7 +63,8 @@ def main() -> int:
|
||||
if args.contract_deadline else None)
|
||||
record = await_intent.normalize_action(
|
||||
action, model_deadline=model_deadline,
|
||||
user_deadline=user_deadline, contract_deadline=contract_deadline)
|
||||
user_deadline=user_deadline, contract_deadline=contract_deadline,
|
||||
task_id=args.task_id)
|
||||
await_intent.emit_normalized_record(record)
|
||||
print(json.dumps(record, sort_keys=True))
|
||||
return 0
|
||||
|
||||
@@ -315,6 +315,7 @@ def _set_cognition_state(op: str, state: str) -> None:
|
||||
def run(*, sha: str, deadline_secs: int,
|
||||
watchdog_secs: int = 300,
|
||||
op_type: str = DEFAULT_OP_TYPE,
|
||||
task_id: str = "",
|
||||
daemon_url: str = ew.DEFAULT_DAEMON_URL) -> int:
|
||||
"""T21.5 §3: ONE LOGICAL AWAIT.
|
||||
|
||||
@@ -401,7 +402,11 @@ def run(*, sha: str, deadline_secs: int,
|
||||
watchdog_secs=min(watchdog_secs, deadline_secs),
|
||||
continuation_state="WAITING_EXTERNAL",
|
||||
op_type=op_type,
|
||||
authority=spec["authority"])
|
||||
authority=spec["authority"],
|
||||
task_id=task_id, # §9: bind the operation to the calling task
|
||||
resume_mode="self") # §3: waiter owns its continuation via
|
||||
# completion-file/inotify; the daemon has NO
|
||||
# session-delivery role for this wait
|
||||
registered_at = rec["created_at"]
|
||||
_set_cognition_state(op, "WAITING_EXTERNAL") # cognition parked
|
||||
deadline = time.monotonic() + deadline_secs
|
||||
@@ -489,6 +494,32 @@ def run(*, sha: str, deadline_secs: int,
|
||||
_set_cognition_state(op, "RESUMABLE" if verdict != "pending"
|
||||
else "RECOVERY_REQUIRED")
|
||||
verdict_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
# §2 STATE OWNERSHIP: the WAITER owns the OPERATION domain. On a
|
||||
# terminal authority verdict, stamp the durable record terminal with
|
||||
# the evidence — the daemon's session-delivery machinery must never
|
||||
# (and can no longer) overwrite this truth. Also updates the parked
|
||||
# continuation_state so fresh-process recovery sees reality.
|
||||
try:
|
||||
rec2 = ew.operation_wait(op) or rec
|
||||
if verdict != "pending":
|
||||
rec2["status"] = "COMPLETED" if verdict == "success" else "FAILED"
|
||||
rec2["verdict"] = verdict
|
||||
rec2["resume_evidence"] = {
|
||||
"kind": "authority_verified_terminal",
|
||||
"verdict": verdict,
|
||||
"mode": mode,
|
||||
"authority": spec["authority"],
|
||||
"sha": sha,
|
||||
"op_type": op_type,
|
||||
"verdict_at": verdict_at,
|
||||
}
|
||||
rec2["ext"] = {**rec2.get("ext", {}),
|
||||
"continuation_state": ("RESUMABLE" if verdict != "pending"
|
||||
else "RECOVERY_REQUIRED")}
|
||||
ew._load_waitstore().save(rec2)
|
||||
except Exception as _e: # noqa: BLE001 — record stamping must never corrupt the envelope (the envelope + authority remain the truth)
|
||||
print(f"ci_event_wait: terminal record stamp failed: {_e}",
|
||||
file=sys.stderr)
|
||||
out = {
|
||||
"sha": sha,
|
||||
"op_type": op_type,
|
||||
@@ -506,17 +537,41 @@ def run(*, sha: str, deadline_secs: int,
|
||||
print(json.dumps(out))
|
||||
return 0 if verdict == "success" else 1
|
||||
|
||||
KNOWN_FLAGS = ("--sha", "--deadline", "--op-type", "--watchdog", "--task-id")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""§5 CLI CONTRACT: no silent argument loss. Every flag is either
|
||||
explicitly supported (and takes effect) or loudly rejected — an
|
||||
acceptance envelope must reflect the configuration actually used."""
|
||||
sha = os.environ.get("DEPLOY_SHA", "")
|
||||
args = sys.argv[1:]
|
||||
if "--sha" in args:
|
||||
sha = args[args.index("--sha") + 1]
|
||||
deadline = 900
|
||||
if "--deadline" in args:
|
||||
deadline = int(args[args.index("--deadline") + 1])
|
||||
op_type = DEFAULT_OP_TYPE
|
||||
if "--op-type" in args:
|
||||
op_type = args[args.index("--op-type") + 1]
|
||||
watchdog = 300
|
||||
task_id = ""
|
||||
i = 0
|
||||
while i < len(args):
|
||||
a = args[i]
|
||||
if a == "--sha":
|
||||
sha = args[i + 1]; i += 2
|
||||
elif a == "--deadline":
|
||||
deadline = int(args[i + 1]); i += 2
|
||||
elif a == "--op-type":
|
||||
op_type = args[i + 1]; i += 2
|
||||
elif a == "--watchdog":
|
||||
watchdog = int(args[i + 1]); i += 2
|
||||
elif a == "--task-id":
|
||||
task_id = args[i + 1]; i += 2
|
||||
elif a.startswith("--"):
|
||||
print(json.dumps({"error": f"unknown flag {a!r}",
|
||||
"known_flags": list(KNOWN_FLAGS),
|
||||
"contract": "no silent argument loss"}))
|
||||
return 2
|
||||
else:
|
||||
print(json.dumps({"error": f"unexpected positional {a!r}",
|
||||
"known_flags": list(KNOWN_FLAGS)}))
|
||||
return 2
|
||||
if not sha:
|
||||
print(json.dumps({"error": "no sha (DEPLOY_SHA or --sha required)"}))
|
||||
return 1
|
||||
@@ -525,7 +580,8 @@ def main() -> int:
|
||||
# got a silent "pending" verdict forever because the var was never
|
||||
# exported. Export it so the canonical authority can resolve.
|
||||
os.environ["DEPLOY_SHA"] = sha
|
||||
return run(sha=sha, deadline_secs=deadline, op_type=op_type)
|
||||
return run(sha=sha, deadline_secs=deadline, op_type=op_type,
|
||||
watchdog_secs=watchdog, task_id=task_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
"""completion_file.py — REAL_ASYNC_BRIDGE: durable completion-file delivery
|
||||
with inotify wake. Replaces FIFO+lease for process-wait delivery.
|
||||
|
||||
DESIGN:
|
||||
Producer (bridge/daemon/notify):
|
||||
1. Validates operation identity
|
||||
2. Atomically writes a completion file:
|
||||
resumes/<operation_id>.done
|
||||
containing the event/verdict JSON (write to tmp, rename for atomicity)
|
||||
|
||||
Waiter (ci_event_wait / any continuation consumer):
|
||||
1. Checks if the completion file already exists (event-before-waiter)
|
||||
2. If not: registers inotify watch on the resumes/ directory
|
||||
3. Re-checks after watch registration (closes the registration race)
|
||||
4. Blocks in kernel on inotify — zero CPU, zero timer
|
||||
5. On file-created event: reads the file, verifies authority, returns
|
||||
|
||||
The event is a WAKE SIGNAL. Authority still decides terminal truth.
|
||||
The completion file is durable — survives daemon and waiter restarts.
|
||||
|
||||
WHY NOT FIFO:
|
||||
FIFO is one-shot (token consumed), requires lease management, and
|
||||
fails under session-busy guards. A completion file is atomic, durable,
|
||||
and works across restarts. inotify wakes the process with zero CPU
|
||||
cost and zero latency between file creation and wake.
|
||||
|
||||
WHY NOT POLLING:
|
||||
Polling introduces an arbitrary interval. inotify is a kernel callback
|
||||
that fires exactly when the filesystem changes — the latency tracks
|
||||
the real completion event, not a configured interval.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import struct
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
def _wr_bridge() -> Path:
|
||||
# WR_ROOT = daemon/infra convention; RUNE_WR_ROOT = repo/scripts
|
||||
# convention. Both honored (WR_ROOT wins) so the daemon, the waiter,
|
||||
# and any isolated test root always resolve the SAME directory.
|
||||
root = (os.environ.get("WR_ROOT")
|
||||
or os.environ.get("RUNE_WR_ROOT")
|
||||
or "/mnt/workdrive/wr-bridge")
|
||||
return Path(root)
|
||||
|
||||
def _resumes_dir() -> Path:
|
||||
return _wr_bridge() / "resumes"
|
||||
|
||||
# inotify constants (Linux)
|
||||
IN_CREATE = 0x100
|
||||
IN_CLOSE_WRITE = 0x8
|
||||
IN_MOVED_TO = 0x80
|
||||
|
||||
|
||||
def completion_path(operation_id: str) -> Path:
|
||||
return _resumes_dir() / f"{operation_id}.done"
|
||||
|
||||
|
||||
def write_completion(operation_id: str, event: dict) -> Path:
|
||||
"""Atomically write a durable completion file. The rename(2) ensures
|
||||
the waiter never sees a partial file."""
|
||||
_resumes_dir().mkdir(parents=True, exist_ok=True)
|
||||
target = completion_path(operation_id)
|
||||
tmp = target.parent / f".{target.name}.tmp.{os.getpid()}"
|
||||
tmp.write_text(json.dumps(event, sort_keys=True) + "\n")
|
||||
os.rename(str(tmp), str(target)) # atomic on same filesystem
|
||||
os.chmod(str(target), 0o666) # cross-user bridge
|
||||
return target
|
||||
|
||||
|
||||
def read_completion(operation_id: str) -> dict | None:
|
||||
"""Read a durable completion file. Returns None if absent."""
|
||||
p = completion_path(operation_id)
|
||||
if not p.is_file():
|
||||
return None
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def completion_exists(operation_id: str) -> bool:
|
||||
return completion_path(operation_id).is_file()
|
||||
|
||||
|
||||
def _inotify_init_and_add_watch(directory: str) -> int:
|
||||
"""Create an inotify instance and add a watch on the directory.
|
||||
Returns the inotify fd."""
|
||||
libc = __import__("ctypes").CDLL(None, use_errno=True)
|
||||
fd = libc.inotify_init()
|
||||
if fd < 0:
|
||||
raise OSError("inotify_init failed")
|
||||
# IN_CREATE | IN_MOVED_TO: catch both direct creation and atomic rename
|
||||
mask = IN_CREATE | IN_MOVED_TO
|
||||
wd = libc.inotify_add_watch(fd, directory.encode(), mask)
|
||||
if wd < 0:
|
||||
os.close(fd)
|
||||
raise OSError(f"inotify_add_watch failed for {directory}")
|
||||
return fd
|
||||
|
||||
|
||||
def _inotify_read(fd: int, timeout_secs: float | None = None) -> list[str]:
|
||||
"""Read pending inotify events. Returns list of filenames that were
|
||||
created/moved-to. Blocks until at least one event or timeout."""
|
||||
poll = select.poll()
|
||||
poll.register(fd, select.POLLIN)
|
||||
events = []
|
||||
buf = os.read(fd, 4096)
|
||||
pos = 0
|
||||
while pos < len(buf):
|
||||
# struct: wd(i32), mask(u32), cookie(u32), name_len(u32)
|
||||
wd, mask, cookie, name_len = struct.unpack_from("iIII", buf, pos)
|
||||
pos += 16
|
||||
name = buf[pos:pos + name_len].rstrip(b"\x00").decode("utf-8", "replace")
|
||||
pos += name_len
|
||||
if mask & (IN_CREATE | IN_MOVED_TO):
|
||||
events.append(name)
|
||||
return events
|
||||
|
||||
|
||||
def wait_for_completion(operation_id: str, *,
|
||||
timeout_secs: float | None = None,
|
||||
poll_callback=None) -> dict | None:
|
||||
"""Block until a completion file exists for this operation.
|
||||
|
||||
Uses inotify for zero-latency, zero-CPU wake. Falls back to
|
||||
polling_callback (if provided) when inotify is unavailable.
|
||||
|
||||
The completion file is a WAKE SIGNAL. The caller must still verify
|
||||
canonical authority after reading it.
|
||||
|
||||
Returns the event dict from the completion file, or None on timeout.
|
||||
"""
|
||||
target = completion_path(operation_id)
|
||||
|
||||
# 1. check-before-watch (event may have arrived before we started)
|
||||
existing = read_completion(operation_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
_resumes_dir().mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
fd = _inotify_init_and_add_watch(str(_resumes_dir()))
|
||||
except OSError:
|
||||
# inotify unavailable: fall back to polling callback
|
||||
if poll_callback:
|
||||
return poll_callback(operation_id, timeout_secs)
|
||||
return None
|
||||
|
||||
try:
|
||||
# 2. re-check after watch registration (closes the race between
|
||||
# the initial check and the watch being active)
|
||||
existing = read_completion(operation_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
# 3. block in kernel on inotify — zero CPU, exact latency
|
||||
import select as _select
|
||||
poll = _select.poll()
|
||||
poll.register(fd, _select.POLLIN)
|
||||
deadline = (time.monotonic() + timeout_secs) if timeout_secs else None
|
||||
|
||||
while True:
|
||||
remaining = None
|
||||
if deadline is not None:
|
||||
remaining = max(0.1, deadline - time.monotonic())
|
||||
if remaining <= 0.1:
|
||||
return None # timeout
|
||||
events = poll.poll(remaining * 1000 if remaining else None)
|
||||
if not events:
|
||||
continue # poll timeout, re-check
|
||||
names = _inotify_read(fd)
|
||||
if target.name in names:
|
||||
# 4. file appeared — read it
|
||||
existing = read_completion(operation_id)
|
||||
if existing is not None:
|
||||
return existing
|
||||
# partial/corrupt: keep watching
|
||||
finally:
|
||||
os.close(fd)
|
||||
return None
|
||||
|
||||
|
||||
def clear_completion(operation_id: str) -> None:
|
||||
"""Remove a consumed completion file (idempotent cleanup)."""
|
||||
p = completion_path(operation_id)
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""install_mimo_tools.py — canonical MiMo tool installer (TTY-repair tranche).
|
||||
|
||||
Canonical sources: config/mimo-tools/{bash.ts,await-operation.ts}
|
||||
Targets: ~/.mimocode/tools/{bash.ts,await-operation.ts}
|
||||
Verifies hashes after install; detects drift in both directions
|
||||
(the parity test reads these hashes).
|
||||
"""
|
||||
import hashlib
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
TOOLS = ["bash.ts", "await-operation.ts"]
|
||||
|
||||
|
||||
def sha256(p: Path) -> str:
|
||||
return hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
rc = 0
|
||||
for name in TOOLS:
|
||||
canon = REPO / "config" / "mimo-tools" / name
|
||||
target = Path.home() / ".mimocode" / "tools" / name
|
||||
if not canon.is_file():
|
||||
print(f"{name}: canonical source missing: {canon}")
|
||||
rc = 1
|
||||
continue
|
||||
canon_hash = sha256(canon)
|
||||
if target.is_file() and sha256(target) == canon_hash:
|
||||
print(f"{name}: in-sync {canon_hash[:12]}")
|
||||
continue
|
||||
drift = f"DRIFT canonical={canon_hash[:12]}"
|
||||
if target.is_file():
|
||||
drift += f" installed={sha256(target)[:12]}"
|
||||
else:
|
||||
drift += " installed=MISSING"
|
||||
print(f"{name}: {drift} — installing")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(canon, target)
|
||||
if sha256(target) != canon_hash:
|
||||
print(f"{name}: INSTALL FAILED (hash mismatch after copy)")
|
||||
rc = 1
|
||||
else:
|
||||
print(f"{name}: installed {canon_hash[:12]}")
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""install_wr_daemon.py — REAL_ASYNC_BRIDGE §6: daemon governance.
|
||||
|
||||
Canonical source lives IN THIS REPO at bridge/daemon/. The active install
|
||||
is ~/.local/share/mimo-wait-resume/. This script is the ONLY supported
|
||||
way to move code between them:
|
||||
|
||||
- copy canonical source → install dir (atomic, per file)
|
||||
- verify sha256 convergence (fail loud on divergence)
|
||||
- optionally restart the daemon via scripts/start_wr_daemon.py
|
||||
|
||||
Never hand-edit the installed copy: edit the repo source, install, restart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
CANONICAL = REPO / "bridge" / "daemon"
|
||||
INSTALL = Path.home() / ".local/share/mimo-wait-resume"
|
||||
FILES = ("daemon.py", "waitstore.py", "verify.py", "completion_file.py")
|
||||
|
||||
|
||||
def sha256(p: Path) -> str:
|
||||
return hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--check", action="store_true", help="verify convergence only; no writes")
|
||||
ap.add_argument("--restart", action="store_true", help="restart the daemon after install")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not CANONICAL.is_dir():
|
||||
print(f"canonical source missing: {CANONICAL}")
|
||||
return 1
|
||||
INSTALL.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
diverged = []
|
||||
for name in FILES:
|
||||
src, dst = CANONICAL / name, INSTALL / name
|
||||
if args.check:
|
||||
if not dst.is_file() or sha256(src) != sha256(dst):
|
||||
diverged.append(name)
|
||||
continue
|
||||
if not dst.is_file() or sha256(src) != sha256(dst):
|
||||
tmp = dst.with_suffix(f".tmp.{name}")
|
||||
shutil.copy2(src, tmp)
|
||||
tmp.replace(dst) # atomic
|
||||
print(f"installed: {name} {sha256(src)[:12]}")
|
||||
else:
|
||||
print(f"unchanged: {name} {sha256(src)[:12]}")
|
||||
|
||||
if args.check:
|
||||
if diverged:
|
||||
print(f"DIVERGED: {', '.join(diverged)} — run install_wr_daemon.py")
|
||||
return 1
|
||||
print("CONVERGED: installed copy == canonical repo source")
|
||||
return 0
|
||||
|
||||
# post-install convergence proof
|
||||
bad = [n for n in FILES if sha256(CANONICAL / n) != sha256(INSTALL / n)]
|
||||
if bad:
|
||||
print(f"INSTALL FAILED convergence: {bad}")
|
||||
return 1
|
||||
print("CONVERGED: installed copy == canonical repo source")
|
||||
|
||||
if args.restart:
|
||||
rc = subprocess.call([sys.executable, str(REPO / "scripts" / "start_wr_daemon.py")])
|
||||
if rc != 0:
|
||||
print("restart failed")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -7,6 +7,13 @@ The event is a WAKE, never completion truth: the waiting controller
|
||||
deploy_qualify.py after resuming. Best-effort by design — if the bridge
|
||||
is unreachable the waiter's watchdog still recovers truthfully (that is
|
||||
the §16D lost-event contract), so a notify failure must NEVER fail CI.
|
||||
|
||||
REAL_ASYNC_BRIDGE (R3): the completion event is ALSO written directly as
|
||||
a durable completion file (resumes/<operation_id>.done). Delivery no
|
||||
longer depends on the daemon being alive: the waiter blocks on inotify
|
||||
for the file itself (kernel wake, zero latency, zero CPU), and the
|
||||
durable file survives daemon AND waiter restarts (R9). The write is
|
||||
best-effort on the same terms as the POST — never fails CI.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -35,6 +42,42 @@ def main() -> int:
|
||||
"exit_code": 0 if result == "success" else 1,
|
||||
"completed_at": time.time(),
|
||||
}
|
||||
# R8 lost-event failure-recovery: operation-scoped fault injection.
|
||||
# A rule keyed by the EXACT operation_id suppresses BOTH normal
|
||||
# delivery channels (durable completion file AND bridge POST), so the
|
||||
# waiter's watchdog/failure-detector path is the only route to truth.
|
||||
# Disabled by default (no rule file => zero behavior change) and
|
||||
# observable: every decision is recorded in the R8 fault log.
|
||||
try:
|
||||
import r8_fault # noqa: E402 — sibling module
|
||||
if r8_fault.should_suppress(payload["operation_id"]):
|
||||
r8_fault.record(payload["operation_id"], "suppressed",
|
||||
{"sha": sha, "exit_code": payload["exit_code"],
|
||||
"source_terminal_at": payload["completed_at"]})
|
||||
print("notify-event-bridge: R8 fault injection — completion "
|
||||
f"event intentionally suppressed for "
|
||||
f"{payload['operation_id']} (no .done file, no POST); "
|
||||
"watchdog authority recovery is the expected path",
|
||||
flush=True)
|
||||
return 0
|
||||
r8_fault.record(payload["operation_id"], "delivered_normally",
|
||||
{"sha": sha, "rule_active": False})
|
||||
except Exception as exc: # noqa: BLE001 — fault seam must never break delivery
|
||||
print(f"notify-event-bridge: R8 fault-seam probe failed ({exc}); "
|
||||
"continuing with normal delivery", flush=True)
|
||||
# R3 durable delivery: write the completion file FIRST — it works even
|
||||
# if the bridge daemon is down, and the atomic rename means any waiter
|
||||
# (inotify-watching or starting later) sees a complete file or none.
|
||||
try:
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import completion_file as cf # noqa: E402 — sibling module
|
||||
written = cf.write_completion(payload["operation_id"], payload)
|
||||
print(f"notify-event-bridge: durable completion file {written}",
|
||||
flush=True)
|
||||
except Exception as exc: # noqa: BLE001 — delivery is best-effort; watchdog recovery covers a lost completion file (§16D contract)
|
||||
print(f"notify-event-bridge: completion-file write failed ({exc}); "
|
||||
f"bridge POST and watchdog recovery still cover delivery",
|
||||
flush=True)
|
||||
for attempt in range(2):
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""r8_fault.py — REAL_ASYNC_BRIDGE R8: operation-scoped lost-event fault
|
||||
injection (evidence-only harness).
|
||||
|
||||
DESIGN CONTRACT (R8 §3):
|
||||
disabled by default no rule file -> zero behavior change
|
||||
operation-scoped rule keyed by EXACT operation_id
|
||||
reversible delete the rule file / rule entry
|
||||
observable every decision appended to a JSONL log
|
||||
cannot affect other waits rule file checked by operation_id only
|
||||
|
||||
The ONLY consumer is scripts/notify_event_bridge.py: before normal
|
||||
delivery it consults should_suppress(operation_id). Suppression means:
|
||||
- NO durable completion file is written
|
||||
- NO bridge POST (no /process-event wake)
|
||||
- the suppression decision is recorded as R8 fault evidence
|
||||
Nothing else in the stack reads the rule. The daemon, waitstore,
|
||||
authority verification, and completion-file machinery are untouched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
RUNE_WR_ROOT = os.environ.get("RUNE_WR_ROOT") or os.environ.get("WR_ROOT") or "/mnt/workdrive/wr-bridge"
|
||||
RULES_PATH = Path(RUNE_WR_ROOT) / "r8_suppress_rules.json"
|
||||
FAULT_LOG = Path(RUNE_WR_ROOT) / "r8_fault_log.jsonl"
|
||||
|
||||
REASON = "event intentionally suppressed for R8 lost-event failure-recovery"
|
||||
|
||||
|
||||
def _rules() -> dict:
|
||||
"""Read the rule file. Absent/unreadable => no suppression (fail-open
|
||||
to NORMAL delivery; the fault must never take the healthy path down)."""
|
||||
try:
|
||||
return json.loads(RULES_PATH.read_text())
|
||||
except Exception: # noqa: BLE001 — absent rule file is the default state
|
||||
return {}
|
||||
|
||||
|
||||
def lookup(operation_id: str) -> dict | None:
|
||||
"""Return the active suppression rule for EXACTLY this operation_id,
|
||||
or None. Expired rules are ignored (not deleted — deletion is the
|
||||
operator's explicit disable action)."""
|
||||
spec = _rules()
|
||||
if not spec.get("enabled"):
|
||||
return None
|
||||
rule = spec.get("operations", {}).get(operation_id)
|
||||
if not rule:
|
||||
return None
|
||||
expires = rule.get("expires_at")
|
||||
if expires:
|
||||
try:
|
||||
if time.time() > time.mktime(time.strptime(expires, "%Y-%m-%dT%H:%M:%SZ")) - time.timezone:
|
||||
return None
|
||||
except ValueError:
|
||||
return None
|
||||
return rule
|
||||
|
||||
|
||||
def should_suppress(operation_id: str) -> bool:
|
||||
return lookup(operation_id) is not None
|
||||
|
||||
|
||||
def record(operation_id: str, stage: str, extra: dict | None = None) -> None:
|
||||
"""Append a durable fault-evidence line (best-effort, never raises)."""
|
||||
try:
|
||||
entry = {
|
||||
"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"operation_id": operation_id,
|
||||
"reason": REASON,
|
||||
"stage": stage,
|
||||
**(extra or {}),
|
||||
}
|
||||
FAULT_LOG.parent.mkdir(parents=True, exist_ok=True)
|
||||
with FAULT_LOG.open("a") as f:
|
||||
f.write(json.dumps(entry, sort_keys=True) + "\n")
|
||||
except OSError as exc: # noqa: BLE001 — evidence logging must never break CI
|
||||
print(f"r8_fault: evidence log write failed: {exc}", file=sys.stderr)
|
||||
@@ -6,7 +6,7 @@ This is the main entry point for the Rune program.
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
__version__ = "0.85.9"
|
||||
__version__ = "0.86.0"
|
||||
|
||||
# One build identity (handoff 2026-09-10 §4): the semantic version above is
|
||||
# the repo-level truth (auto-bumped at merge); the DEPLOYED revision (exact
|
||||
|
||||
@@ -231,7 +231,7 @@ def main() -> int:
|
||||
)
|
||||
check(
|
||||
"Rune minor version advertises provisioning and restore verification",
|
||||
'__version__ = "0.85.9"' in rune_version,
|
||||
'__version__ = "0.86.0"' in rune_version,
|
||||
failures,
|
||||
)
|
||||
# #169 policy update: auto-bump on merge is sanctioned (Nick directive:
|
||||
|
||||
+28
-4
@@ -175,7 +175,8 @@ def register_process_wait(operation_id: str, *, contract_id: str = "",
|
||||
continuation_state: str = "WAITING_EXTERNAL",
|
||||
op_type: str = "",
|
||||
authority: str = "",
|
||||
task_id: str = "") -> dict:
|
||||
task_id: str = "",
|
||||
resume_mode: str = "fifo") -> dict:
|
||||
"""E-PATCH §4: the PendingOperation IS the durable continuation object.
|
||||
|
||||
continuation_state is first-class cognitive quiescence:
|
||||
@@ -183,7 +184,18 @@ def register_process_wait(operation_id: str, *, contract_id: str = "",
|
||||
op_type/authority record the CANONICAL execution semantics (derived by
|
||||
intent normalization, never model-supplied). task_id/project bind the
|
||||
operation to the current programme so fresh-process recovery can
|
||||
resolve relevance without the model reconstructing state."""
|
||||
resolve relevance without the model reconstructing state.
|
||||
|
||||
resume_mode owns the SESSION-DELIVERY domain (REAL_ASYNC_BRIDGE §2/§3):
|
||||
"fifo" — legacy delivery: the daemon writes a resume token to the
|
||||
operation's FIFO on event match (T15 semantics; the waiter
|
||||
blocks reading the FIFO). Default preserves T15 builder
|
||||
behavior.
|
||||
"self" — the waiter process itself blocks on the durable completion
|
||||
file (inotify) and stamps its own terminal operation state
|
||||
on authority verification. NO resume_cmd is registered: the
|
||||
daemon must never dispatch a delivery worker for this wait,
|
||||
and a dead reader cannot downgrade operation truth."""
|
||||
"""Register a durable PROCESS_EVENT wait for one operation. The
|
||||
per-wait resume_cmd writes a resume token to the operation's FIFO —
|
||||
the bridge can resume a Python controller, not just MiMo sessions.
|
||||
@@ -208,6 +220,13 @@ def register_process_wait(operation_id: str, *, contract_id: str = "",
|
||||
"fd=os.open(sys.argv[1], os.O_WRONLY | os.O_NONBLOCK);"
|
||||
"os.write(fd, json.dumps({'wait_id': sys.argv[2],"
|
||||
" 'resumed_at': time.time()}).encode());os.close(fd)")
|
||||
if resume_mode == "self":
|
||||
# REAL_ASYNC_BRIDGE §3: no session-delivery worker. The waiter
|
||||
# self-resumes on the durable completion file; the daemon's only
|
||||
# roles are event persistence and failure-detection sweeps.
|
||||
resume_cmd = None
|
||||
else:
|
||||
resume_cmd = ["python3", "-c", writer, str(fifo), "{wait_id}"]
|
||||
rec = ws.new_wait(
|
||||
session_id=f"controller-{operation_id}",
|
||||
provider=provider,
|
||||
@@ -224,7 +243,7 @@ def register_process_wait(operation_id: str, *, contract_id: str = "",
|
||||
"independent verification",
|
||||
on_success="continue programme continuation canary",
|
||||
on_failure="classify failure; recovery or block",
|
||||
resume_cmd=["python3", "-c", writer, str(fifo), "{wait_id}"],
|
||||
resume_cmd=resume_cmd,
|
||||
)
|
||||
rec["ext"] = {"contract_id": contract_id, "operation_id": operation_id,
|
||||
"fifo": str(fifo),
|
||||
@@ -232,7 +251,12 @@ def register_process_wait(operation_id: str, *, contract_id: str = "",
|
||||
or int(os.environ.get("WR_WATCH_SECS", "600")),
|
||||
"continuation_state": continuation_state,
|
||||
"op_type": op_type, "authority": authority,
|
||||
"task_id": task_id}
|
||||
"task_id": task_id,
|
||||
# §3 delivery-domain marker: the daemon drive loop skips
|
||||
# waits marked "self" (the waiter owns its continuation);
|
||||
# records WITHOUT the marker remain eligible for the
|
||||
# env-template session-delivery path (WR_RESUME_CMD).
|
||||
"resume_mode": resume_mode}
|
||||
ws.save(rec)
|
||||
# cross-user bridge (T16 §16C live finding): the daemon reconciler may
|
||||
# run under a different account than the registering controller — the
|
||||
|
||||
@@ -8,7 +8,7 @@ mode system, subject header, shared memory and identity.
|
||||
|
||||
This is not an interface. This is how I see, speak, and interact with the world.
|
||||
|
||||
Version: v0.85.9
|
||||
Version: v0.86.0
|
||||
Features: 41 (24 original + 17 new native commands)
|
||||
|
||||
Usage:
|
||||
@@ -1885,7 +1885,7 @@ class RuneApp(App):
|
||||
# Show version info
|
||||
self.chat_channel.add_message(
|
||||
"system",
|
||||
f"[dim]Rune v0.85.9 | Textual TUI | {len(loaded)} native modules[/dim]",
|
||||
f"[dim]Rune v0.86.0 | Textual TUI | {len(loaded)} native modules[/dim]",
|
||||
)
|
||||
|
||||
# Auto-show night mode suggestion if evening/night hours
|
||||
|
||||
@@ -6,3 +6,16 @@
|
||||
{"action": "ContinuePendingOperation", "at": "2026-09-17T23:19:18Z", "authority": "ci_runs", "deadline_provenance": "GOVERNED_OPERATION_POLICY", "deadline_secs": 1800, "matched_by": "sleep_poll_intent", "model_deadline_ignored": true, "model_op_type_ignored": false, "normalized": true, "op_type": "branch_ci", "operation_id": "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "raw_command_kind": "sleep_poll", "raw_sleep_seconds_suppressed": 150}
|
||||
{"action": "ContinuePendingOperation", "at": "2026-09-18T10:30:10Z", "authority": "ci_runs", "deadline_provenance": "GOVERNED_OPERATION_POLICY", "deadline_secs": 1800, "evidence": "commit_on_feature_branch", "model_deadline_ignored": true, "model_op_type_ignored": true, "normalized": true, "op_type": "branch_ci", "operation_id": "ci-427c3ac6bb3fd67492d3596a33919d1e5aa4a8d1"}
|
||||
{"action": "ContinuePendingOperation", "at": "2026-09-18T10:30:40Z", "authority": "ci_runs", "deadline_provenance": "GOVERNED_OPERATION_POLICY", "deadline_secs": 1800, "evidence": "commit_on_feature_branch", "model_deadline_ignored": true, "model_op_type_ignored": false, "normalized": true, "op_type": "branch_ci", "operation_id": "ci-427c3ac6bb3fd67492d3596a33919d1e5aa4a8d1", "raw_command_kind": "sleep_poll", "raw_sleep_seconds_suppressed": 240}
|
||||
{"at": "2026-09-18T20:23:58Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T04:42:44Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T04:44:06Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T04:46:33Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T04:49:08Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T04:52:18Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T04:54:51Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T04:57:26Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T04:59:46Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T05:02:08Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"action": "ContinuePendingOperation", "at": "2026-09-19T05:16:37Z", "authority": "ci_runs", "deadline_provenance": "GOVERNED_OPERATION_POLICY", "deadline_secs": 1800, "evidence": "explicit_id_fresh_registration", "model_deadline_ignored": false, "model_op_type_ignored": false, "normalized": true, "op_type": "branch_ci", "operation_id": "ci-c6c18ae041ddac41e714af14ca0ce8c2e918078c"}
|
||||
{"at": "2026-09-19T05:18:31Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
{"at": "2026-09-19T05:20:46Z", "candidates": ["", "", "", "", "", "", "", "", "", "", "", "", "", "op-latency-proof", "dbg-op", "dbg3", "t15-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-resume-BUILDER-2026-09-15T02:18:49Z-Verboo Code/glm-5.3-flash", "t15-BUILDER-2026-09-15T03:11:05Z-Verboo Code/glm-5.3-flash", "t15-release-196cf19", "", "", "ci-c707a81933f7f90eaaae4000282674b6a2e3cb4c", "ci-4a06d9f11a722b7aa32fc38ab92b3ba8130414af", "ci-b51afea0cbfe00b5493647ed61d071fe104b306a", "", "ci-986b450bc22e969cbd2f09fdcfbc24e4337740ba", "ci-2a4f9c8367d06acbda4b39c1cee388796b25cc85", "ci-fe9d154778c117e8b0f1fbae4e5e7d9121546839", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-c9dee14c626b853cab625b431af3b23866b84314", "ci-repro-test-sha", "", "", "", "", "", "", "", "", "", "", "ci-3c54782634993361bbab3c66f948fc59121865a5", "ci-3c54782634993361bbab3c66f948fc59121865a5", "", "", "", "ci-dcead3feccb57b6dfb30effd96fcac6b563f8099", "ci-ab2dfe7b09c154a329c5145da4d32639a83579b6", "ci-95946b6efd883e56797134617d42daed1acf798a", "ci-45869a77cdea1c2ce9e5c1f54c9840c3acef1487", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-98cb3a98a5f2e245cd0fd6003ffd02a238120ddc", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-2ccac89ec448900766d47e322b2723da5b2ce9dd", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-824985488196ba5e5033318b7b2b6cdc82719ed6", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-9e903dacafdbc40dccf17ffb3ea09e8452396860", "ci-5d20dc7f513b740a165952ca00f758660d328c43", "ci-0a470d86b80baefeb71c795626a901fac19bf731", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-116d1ffbeb49c879855b982c89d51db25528537d", "ci-40b3731928f499a77653613cb6c3e6db93cb64a1", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-be6bc17e93d8ffbc2679d913e55dd848dc8fa82e", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-f3ef4672f3209e73bd110fc49efff9834a2eb987", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-6b5d8ad02308f24422b1efd7c0d343ebb5c7a19d", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-ba2966e6318d6f3f1d2260732b1534732d650c0c", "ci-00013f8c0b174011bcea12351facf7f1f382e8a3", "ci-756ebe6f45ad83a919738539a1565c17ed8572ac", "ci-df8262bd58122e33d4c447dc863e39ddde684963", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-4944f9e79d0a9c4df1d76a0d8c119e33e1f9a55", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed7b7b0b8c29f1d27c4d328f4a3a2d1b61", "ci-f545f1ed8d5cf2ae99554eb02b95771082e449c8", "ci-7cf7c8d0a9c4aa7c6c1a7c2e0a2e7c3b4c5d6e7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7cf7c8dd8d82ef2165aedbf456683448a7536bc7", "ci-7008743a1a7f8c1b9e4b0b8e4c4f4f4f4f4f4f4", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-70087436203f8e44181b94676a05418730b8de5d", "ci-26e37653e3f8c1b9e4b0b8e4c4f4f4f4f4f4f4f", "ci-26e3765319bc29064f7d42f092bc86dcc7b2ecc0", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-2b6a5a6b95629d3cf48fd171cbf6a2544f771817", "ci-5db8b202d97e4721369aa507e2bea7b1cf66af5c", "ci-04a277f7668e99d56c1b7c05e2e40a1da93dca55", "ci-5718a3ea85c397434595e79b7a63700617c1de21", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-246f26663d8fa2aba3a16039c86feb13a6d0bd2c", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-f714120", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-65001196d831a7f51b2bbc2b6bbcfe34c70442ff", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-53b4830b08a7dd72a1a7236f1752516880a29bc8", "ci-853cf3e0ae10ad3d45cd879527093368ccad8251", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-599c3ab1c23f7eaec323c67b3523fdb9e1acea74", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd50e77ac55db4056951bd6655a0632", "ci-a54ab559cbd59bb9696d329c998cd58a221edd72", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-1c5dc557a69a368aad267d1871f4eda646a822fb", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-05f2a74c6fa086ddec55d146b780859df34d31a5", "ci-2e05ea6", "ci-d53bb9a", "ci-d53bb9a53Line truncated
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"tranche": "REAL_ASYNC_BRIDGE",
|
||||
"test_class": "FAILURE-RECOVERY (deliberately separate from healthy-path statistics)",
|
||||
"status": "SPECIFIED_NOT_EXECUTED",
|
||||
"principle": "suppressing normal event delivery must NOT silently lose the operation; watchdog/reconciliation is failure detection and must recover terminal truth safely",
|
||||
"setup": {
|
||||
"environment": "isolated WR_ROOT (test_wr_v04 pattern) OR live daemon with injected fault \u2014 decide at execution",
|
||||
"operation": "one real or stub operation with a known terminal authority verdict",
|
||||
"fault_injection": "suppress ALL normal delivery paths for the operation: block/redirect the notify producer AND the daemon /process-event completion-file write for that operation_id (e.g., fault shim on completion_file.write_completion + notify POST)"
|
||||
},
|
||||
"required_sequence": [
|
||||
"waiter parks (WAITING_EXTERNAL, inotify subscribed)",
|
||||
"source operation reaches terminal truth (authority would say success) \u2014 delivery suppressed",
|
||||
"watchdog window expires with NO event delivered",
|
||||
"watchdog/reconciler notices abnormality (no event + authority reread)",
|
||||
"authority is consulted directly and returns TERMINAL verdict",
|
||||
"waiter transitions to terminal truth (COMPLETED) with mode=watchdog_authority_recovered",
|
||||
"durable operation state stamped authority_verified_terminal \u2014 truthful",
|
||||
"no silent loss: operation did NOT stay WAITING_EXTERNAL past recovery",
|
||||
"late/duplicate delivery after recovery is idempotent (no double-processing)"
|
||||
],
|
||||
"metrics_labels": {
|
||||
"FAILURE_RECOVERY_WATCHDOG_CYCLES": "count (>=1 expected)",
|
||||
"FAILURE_RECOVERY_AUTHORITY_REREAD_COUNT": "count (>=1 expected)",
|
||||
"FAILURE_RECOVERY_TIME_TO_TERMINAL_TRUTH": "watchdog expiry -> authority verdict",
|
||||
"FAILURE_RECOVERY_SILENT_LOSS": "must be 0",
|
||||
"FAILURE_RECOVERY_STATE_CONTRADICTION": "must be 0"
|
||||
},
|
||||
"explicitly_not_claimed": "this test does NOT measure or validate healthy event-driven completion latency \u2014 that is the accepted final unobserved run's domain",
|
||||
"acceptance": "all FAILURE_RECOVERY_* metrics recorded; SILENT_LOSS=0; STATE_CONTRADICTION=0; terminal truth recovered before deadline containment",
|
||||
"related_gates": [
|
||||
"R8_LOST_EVENT_RECOVERY"
|
||||
],
|
||||
"created_at": "2026-09-19T01:20:00Z"
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
{
|
||||
"tranche": "MIMO_EXECUTION_SEMANTICS",
|
||||
"purpose": "Fix the caller, freeze Rune core \u2014 intent compilation at the MiMo execution boundary",
|
||||
"predecessor": "event_hardening_gates.json (superseded verdict)",
|
||||
"created_at": "2026-09-18T19:57:25Z",
|
||||
"gates": [
|
||||
{
|
||||
"gate_id": "M10_LIVE_MASTER_SINGLE_CONTINUATION",
|
||||
"mandatory": true,
|
||||
"requirement": "master/deploy resolved via one canonical continuation; no model re-arms",
|
||||
"status": "UNRUN",
|
||||
"evidence": "master/deploy flow via model-facing tool not exercised in this tranche",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": ""
|
||||
},
|
||||
{
|
||||
"gate_id": "M11_RUNE_CORE_FROZEN",
|
||||
"mandatory": true,
|
||||
"requirement": "RUNE_CORE_FILES_MODIFIED=0 or non-MiMo reproduction proven for each",
|
||||
"status": "PASS",
|
||||
"evidence": "RUNE core touch = src/rune/event_wait.py resume_mode (additive, default-preserving) with frozen non-MiMo tests; all other core files unmodified in tranche",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M12_CLEAN_WINDOW_AUDIT",
|
||||
"mandatory": true,
|
||||
"requirement": "no manual internal APIs in the acceptance window",
|
||||
"status": "PASS",
|
||||
"evidence": "final unobserved run window 01:01:54..01:10:16Z: zero manual observers, zero cron observers, zero manual internal API calls",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M1_CALLER_ROOT_CAUSE_IDENTIFIED",
|
||||
"mandatory": true,
|
||||
"requirement": "the exact seam where MiMo executes mechanical wait policy instead of compiling intent",
|
||||
"status": "PASS",
|
||||
"evidence": "root cause: model emitted mechanical wait policy at the execution boundary; fix = intent normalization (await_intent) + operation-id-first tool + bash routing; live proof final run",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M2_OPERATION_ID_FIRST_TOOL_SCHEMA",
|
||||
"mandatory": true,
|
||||
"requirement": "await-operation tool: operation_id-first; legacy args accepted as hints only",
|
||||
"status": "PASS",
|
||||
"evidence": "~/.mimocode/tools/await-operation.ts (sha256 894dc91d\u2026): operation_id + task_id first; sha/op_type/deadline optional non-authoritative hints; final run used operation_id only",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M3_TASK_BOUND_OPERATION_RESOLUTION",
|
||||
"mandatory": true,
|
||||
"requirement": "deterministic task->operation binding; model sha/op_type/deadline non-authoritative",
|
||||
"status": "PASS",
|
||||
"evidence": "--task-id through cli+tool; frozen tests TestTaskBinding (bound resolution, unbound reports missing); stored operation wins over model hints. NOTE: live acceptance run used explicit operation_id; task-scoped live resolution proven at unit level",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M4_BASH_INTENT_COMPILATION_ACTIVE",
|
||||
"mandatory": true,
|
||||
"requirement": "bash.ts routes external wait intent to canonical continuation; literal sleep suppressed",
|
||||
"status": "PASS",
|
||||
"evidence": "bash.ts (1054d8e2) routes external wait intent via await_intent_cli route-bash; live routing observed 00:47:00Z; historical entry raw_sleep_seconds_suppressed=240",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M5_LOWER_LEVEL_INTERNALS_EXCLUDED",
|
||||
"mandatory": true,
|
||||
"requirement": "model cannot select ci_event_wait/deploy_qualify/authority helpers as strategies",
|
||||
"status": "PASS",
|
||||
"evidence": "model-facing surface exposes only await-operation/await_intent_cli; ci_event_wait/deploy_qualify not exposed as strategies; final run tool inventory = await-operation only",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M6_OBSERVABILITY_COMPACT_LOG",
|
||||
"mandatory": true,
|
||||
"requirement": "every translation recorded with binding source/provenance",
|
||||
"status": "PASS",
|
||||
"evidence": "every translation recorded in store/await_intent_log.jsonl with normalized/evidence/matched_by/deadline_provenance fields",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M7_LIVE_WRONG_ARGS_PROOF",
|
||||
"mandatory": true,
|
||||
"requirement": "model emits wrong deadline/op_type/sha; harness compiles to bound operation",
|
||||
"status": "PASS",
|
||||
"evidence": "historical live evidence 2026-09-18T10:30:40Z: model deadline ignored (model_deadline_ignored=true), 240s sleep suppressed; final run emitted no wrong args (operation_id only) \u2014 hint-ignoring machinery active in both",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M8_LIVE_BASH_SLEEP_SUPPRESSED",
|
||||
"mandatory": true,
|
||||
"requirement": "bash sleep intent compiled; LITERAL_EXTERNAL_SLEEP_EXECUTED=false",
|
||||
"status": "PASS",
|
||||
"evidence": "historical: raw_sleep_seconds_suppressed=240 with LITERAL_EXTERNAL_SLEEP not executed; live routing of sleep intent observed 00:47:00Z (plain local sleep executes by design; external intent compiles)",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "M9_LIVE_PR_SINGLE_CONTINUATION",
|
||||
"mandatory": true,
|
||||
"requirement": "PR checks resolved via one canonical continuation; no model re-arms",
|
||||
"status": "UNRUN",
|
||||
"evidence": "PR-checks flow via model-facing tool not exercised in this tranche (branch flow proven; PR/master remain)",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T19:57:25Z",
|
||||
"resolved_at": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
{
|
||||
"tranche": "R8_LOST_EVENT_RECOVERY",
|
||||
"question": "If the source operation finishes but its normal completion event is lost, does Rune recover terminal truth through its failure-detection path without losing state, inventing success, or requiring model intervention?",
|
||||
"verdict": "LOST_EVENT_RECOVERY=PASS (live fault-injection proof, attempt 3; attempt 1 additionally proved truthful FAILURE recovery)",
|
||||
"run_identity": {
|
||||
"sha": "c6c18ae041ddac41e714af14ca0ce8c2e918078c",
|
||||
"operation_id": "ci-c6c18ae041ddac41e714af14ca0ce8c2e918078c",
|
||||
"operation_created_at": "2026-09-19T05:12:39Z",
|
||||
"fault_injection_enabled_at": "2026-09-19T05:12:55Z",
|
||||
"fault_injection_disabled_at": "2026-09-19T05:23:30Z",
|
||||
"session_launch_at": "2026-09-19T05:16:24Z",
|
||||
"wait_id": "wr-1789794997-0a3abd",
|
||||
"attempt_1": {
|
||||
"sha": "89b603024ef564c11e090bc683f72e81ee641f70",
|
||||
"wait_id": "wr-1789790509-a6b8db",
|
||||
"outcome": "suppression + watchdog_authority_recovered PROVEN; verdict=failure was TRUTHFUL (CI genuinely red: S110 silent-handler ratchet 75->77, self-inflicted by seam commit dac6e97, fixed in c6c18ae); type-check job failure cause LIKELY_CPU_CONTENTION / CAUSE_UNCONFIRMED — a heavy local test suite ran concurrently on the same desktop and the job exceeded the 600s pyright timeout; plausible but not independently proven (ratchet re-run PASS on identical checkout+venv)"
|
||||
}
|
||||
},
|
||||
"fault_injection_design": {
|
||||
"module": "scripts/r8_fault.py",
|
||||
"producer_seam": "scripts/notify_event_bridge.py guard clause (records suppression, no .done file, no /process-event POST)",
|
||||
"rule_file": "/mnt/workdrive/wr-bridge/r8_suppress_rules.json (deleted after test)",
|
||||
"properties": ["disabled by default (fail-open)", "operation-scoped (exact operation_id)", "reversible (delete file / enabled=false)", "observable (r8_fault_log.jsonl)", "cannot affect unrelated waits"],
|
||||
"evidence_log": "/mnt/workdrive/wr-bridge/r8_fault_log.jsonl"
|
||||
},
|
||||
"timeline": [
|
||||
{"event": "push", "at": "2026-09-19T05:12:39Z", "source": "git push record"},
|
||||
{"event": "fault_injection_enabled_at", "at": "2026-09-19T05:12:55Z", "source": "rule file"},
|
||||
{"event": "session_launch", "at": "2026-09-19T05:16:24Z", "source": "launch record (model pinned Verboo after Xioami 429)"},
|
||||
{"event": "model_tool_call_at", "at": "2026-09-19T05:16:37.154Z", "source": "transcript tool time.start ts 1789794997154"},
|
||||
{"event": "waiter_registered_at", "at": "2026-09-19T05:16:37Z", "source": "wait record created_at (self-mode, watchdog 300)"},
|
||||
{"event": "source_terminal_at", "at": "2026-09-19T05:17:30.565Z", "source": "r8_fault_log suppressed entry completed_at 1789795050.5647, exit_code 0"},
|
||||
{"event": "normal_event_suppressed_at", "at": "2026-09-19T05:17:30Z", "source": "r8_fault_log stage=suppressed"},
|
||||
{"event": "watchdog_fired_at + authority_terminal_verified_at + canonical_state_recovered_at", "at": "2026-09-19T05:21:40Z", "source": "envelope verdict_at + wait record last_updated (FIRST watchdog window, 303s)"},
|
||||
{"event": "await_tool_returned_at", "at": "2026-09-19T05:21:40.051Z", "source": "transcript tool time.end ts 1789795300051, SAME callID call_931aa8dd984e43949152b9ee"},
|
||||
{"event": "next_model_turn_at", "at": "2026-09-19T05:21:47.055Z", "source": "transcript step_start ts 1789795307055"},
|
||||
{"event": "R8_PROOF_COMPLETE_at", "at": "2026-09-19T05:21:48.017Z", "source": "transcript text ts 1789795308017"},
|
||||
{"event": "session_exit_at", "at": "2026-09-19T05:21:48Z", "source": "/tmp/r8_exit3.txt rc=0"},
|
||||
{"event": "fault_disabled_at", "at": "2026-09-19T05:23:30Z", "source": "rule enabled=false"},
|
||||
{"event": "late_event_released_at + late_event_processed_at", "at": "2026-09-19T05:23:48Z", "source": "event 16b286e1578c682a received_at; bridge matched=0; .done written late"}
|
||||
],
|
||||
"envelope": {
|
||||
"mode": "watchdog_authority_recovered",
|
||||
"rearm_reason": "",
|
||||
"internal_rearm_count": 0,
|
||||
"verdict": "success",
|
||||
"wall_secs": 302.787,
|
||||
"event_received_at": "",
|
||||
"controller_resumed_at": ""
|
||||
},
|
||||
"canonical_terminal_state": {
|
||||
"status": "COMPLETED", "verdict": "success", "continuation_state": "RESUMABLE",
|
||||
"resume_evidence_kind": "authority_verified_terminal",
|
||||
"recovery_provenance": {"mode": "watchdog_authority_recovered", "normal_event_missing": true, "authority": "ci_runs"}
|
||||
},
|
||||
"metrics": {
|
||||
"FAILURE_RECOVERY_SOURCE_TERMINAL_COUNT": 1,
|
||||
"FAILURE_RECOVERY_NORMAL_EVENT_SUPPRESSED_COUNT": 1,
|
||||
"FAILURE_RECOVERY_NORMAL_EVENT_WAKE_COUNT": 0,
|
||||
"FAILURE_RECOVERY_WATCHDOG_FIRE_COUNT": 1,
|
||||
"FAILURE_RECOVERY_AUTHORITY_QUERY_COUNT": 1,
|
||||
"FAILURE_RECOVERY_TERMINAL_RECOVERED_COUNT": 1,
|
||||
"FAILURE_RECOVERY_MODEL_REARM_COUNT": 0,
|
||||
"FAILURE_RECOVERY_DUPLICATE_RESUME_COUNT": 0,
|
||||
"FAILURE_RECOVERY_LATE_EVENT_COUNT": 1,
|
||||
"FAILURE_RECOVERY_LATE_EVENT_DUPLICATE_ACTION_COUNT": 0,
|
||||
"FAILURE_RECOVERY_STATE_CONTRADICTION_COUNT": 0,
|
||||
"FAILURE_RECOVERY_SILENT_LOSS_COUNT": 0
|
||||
},
|
||||
"model_call_counts": {
|
||||
"AWAIT_OPERATION_CALL_COUNT": 1, "OTHER_MODEL_TOOL_CALL_COUNT": 0,
|
||||
"MODEL_SHELL_COUNT": 0, "MODEL_SLEEP_COUNT": 0, "MODEL_POLL_COUNT": 0,
|
||||
"MODEL_CRON_COUNT": 0, "MODEL_REARM_COUNT": 0
|
||||
},
|
||||
"late_event_idempotence": {
|
||||
"terminal_remains_terminal": true,
|
||||
"wait_record_untouched_after_recovery": true,
|
||||
"resume_attempts": 0,
|
||||
"bridge_matched": 0,
|
||||
"no_second_model_resume": "session already exited rc=0 before release; transcript ends at R8_PROOF_COMPLETE"
|
||||
},
|
||||
"healthy_path_regression": {
|
||||
"hashes_unchanged": ["scripts/ci_event_wait.py", "scripts/completion_file.py", "bridge/daemon/completion_file.py", "bridge/daemon/daemon.py", "bridge/daemon/waitstore.py", "bridge/daemon/verify.py", "~/.local/share/mimo-wait-resume/* (deployed runtime)"],
|
||||
"notify_event_bridge_change": "guard clause only (R8 seam); delivery path bytes otherwise unchanged",
|
||||
"deterministic_suites": "50 passed (r8 seam 7 + ci_event_wait + event_wait + lint contract + wait guard + single logical await)",
|
||||
"full_suite_on_89b6030": "2202/2203 (single failure = lint-baseline contract test, caused by the S110 seam defect, fixed in c6c18ae)",
|
||||
"lint_ratchet_after_fix": "S110 75=75 PASS",
|
||||
"default_delivery_proof": "isolated WR_ROOT run: no rule file -> delivered_normally + .done written"
|
||||
},
|
||||
"provider_blockers": "none (PROVIDER_GITEA_STATUS_MATERIALIZATION_BLOCKER not invoked)",
|
||||
"raw_evidence_pointers": {
|
||||
"transcript": "/tmp/r8_transcript3.jsonl",
|
||||
"exit_marker": "/tmp/r8_exit3.txt",
|
||||
"wait_record": "/mnt/workdrive/wr-bridge/waits/wr-1789794997-0a3abd.json",
|
||||
"fault_log": "/mnt/workdrive/wr-bridge/r8_fault_log.jsonl",
|
||||
"late_event": "/mnt/workdrive/wr-bridge/events/16b286e1578c682a.json",
|
||||
"late_done_file": "/mnt/workdrive/wr-bridge/resumes/ci-c6c18ae041ddac41e714af14ca0ce8c2e918078c.done",
|
||||
"run_identity": "/mnt/workdrive/wr-bridge/r8/run_identity.json"
|
||||
},
|
||||
"recorded_at": "2026-09-19T05:30:00Z"
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"tranche": "REAL_ASYNC_BRIDGE",
|
||||
"run_class": "DIAGNOSTIC_EVIDENCE_ONLY (contaminated by observer: 2 forensic sleeps, 1 armed collector; superseded by final unobserved run)",
|
||||
"operation_id": "ci-c288f4726d03993bd6c8cc751c9ab0cdb8713057",
|
||||
"chain": {
|
||||
"session_launch_utc": "2026-09-19T00:41:36Z",
|
||||
"await_tool_called_at": "2026-09-19T00:42:06Z",
|
||||
"waiter_registered_at": "2026-09-19T00:42:06Z",
|
||||
"resume_mode": "self (resume_cmd=None)",
|
||||
"source_terminal_at": "2026-09-19T00:46:20.94Z",
|
||||
"completion_event_received_at": "2026-09-19T00:46:20Z",
|
||||
"completion_file_written_at": "2026-09-19T00:46:20Z",
|
||||
"inotify_wake_at": "2026-09-19T00:46:20Z (envelope event_received_at)",
|
||||
"authority_verified_at": "2026-09-19T00:47:38Z",
|
||||
"await_tool_returned_at": "2026-09-19T00:47:38.741Z (transcript tool_use state=completed)",
|
||||
"next_model_turn_at": "2026-09-19T00:48:21.657Z (automatic)",
|
||||
"session_exit": "natural, before 00:59:52Z"
|
||||
},
|
||||
"canonical_operation_state": {
|
||||
"status": "COMPLETED",
|
||||
"verdict": "success",
|
||||
"evidence_kind": "authority_verified_terminal",
|
||||
"mode": "event"
|
||||
},
|
||||
"state_ownership_fix_held": true,
|
||||
"internal_rearm_count": 1,
|
||||
"watchdog_wake": 0,
|
||||
"deadline_wake": 0
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
{
|
||||
"tranche": "REAL_ASYNC_BRIDGE",
|
||||
"artifact_class": "FINAL_UNOBSERVED_ACCEPTANCE_RUN \u2014 canonical healthy-path evidence",
|
||||
"verdict": "REAL_ASYNC_BRIDGE_COMPLETE",
|
||||
"run_identity": {
|
||||
"sha": "5b61f8a8bb8273d0fbe5124dd81a952f208ef41c",
|
||||
"operation_id": "ci-5b61f8a8bb8273d0fbe5124dd81a952f208ef41c",
|
||||
"operation_created_at": "2026-09-19T01:01:36Z",
|
||||
"session_launch_at": "2026-09-19T01:01:54Z",
|
||||
"session_pid": 3137854,
|
||||
"wait_id": "wr-1789779758-4cb2a6"
|
||||
},
|
||||
"raw_timeline": [
|
||||
{
|
||||
"event": "operation_created_at",
|
||||
"at": "2026-09-19T01:01:36Z",
|
||||
"source": "push record (pre-launch)"
|
||||
},
|
||||
{
|
||||
"event": "session_launch",
|
||||
"at": "2026-09-19T01:01:54Z",
|
||||
"source": "launch record"
|
||||
},
|
||||
{
|
||||
"event": "model_tool_call_at",
|
||||
"at": "2026-09-19T01:02:31.165Z",
|
||||
"source": "transcript step_start"
|
||||
},
|
||||
{
|
||||
"event": "normalize (tool internals)",
|
||||
"at": "2026-09-19T01:02:37Z",
|
||||
"source": "store/await_intent_log.jsonl evidence=explicit_id_fresh_registration"
|
||||
},
|
||||
{
|
||||
"event": "waiter_registered_at",
|
||||
"at": "2026-09-19T01:02:38Z",
|
||||
"source": "wait record created_at"
|
||||
},
|
||||
{
|
||||
"event": "watchdog_failure_detector_cycle",
|
||||
"at": "~2026-09-19T01:07:38Z",
|
||||
"source": "envelope rearm_reason (park+300s)",
|
||||
"precision": "SUPPORTED_BUT_COARSE"
|
||||
},
|
||||
{
|
||||
"event": "source_terminal_at",
|
||||
"at": "2026-09-19T01:08:56.59Z",
|
||||
"source": "event payload completed_at (exit_code 0)"
|
||||
},
|
||||
{
|
||||
"event": "completion_event_received_at",
|
||||
"at": "2026-09-19T01:08:56Z",
|
||||
"source": "durable event d6ebf9466ebbd776 received_at"
|
||||
},
|
||||
{
|
||||
"event": "completion_file_written_at",
|
||||
"at": "2026-09-19T01:08:56Z",
|
||||
"source": "resumes/ci-5b61f8a\u2026.done",
|
||||
"precision": "FILESYSTEM_MTIME_EVIDENCE"
|
||||
},
|
||||
{
|
||||
"event": "inotify_wake_at",
|
||||
"at": "2026-09-19T01:08:56Z",
|
||||
"source": "envelope event_received_at + controller_resumed_at (the event that released the blocking wait)",
|
||||
"precision": "second"
|
||||
},
|
||||
{
|
||||
"event": "authority_verified_at",
|
||||
"at": "2026-09-19T01:09:58Z",
|
||||
"source": "envelope verdict_at"
|
||||
},
|
||||
{
|
||||
"event": "await_tool_returned_at",
|
||||
"at": "2026-09-19T01:09:58.397Z",
|
||||
"source": "transcript tool_use state=completed callID=call_d6491c1a337b44f89c41dae1"
|
||||
},
|
||||
{
|
||||
"event": "next_model_turn_at",
|
||||
"at": "2026-09-19T01:10:12.026Z",
|
||||
"source": "transcript step_start (automatic)"
|
||||
},
|
||||
{
|
||||
"event": "CLEAN_PROOF_COMPLETE_at",
|
||||
"at": "2026-09-19T01:10:16.102Z",
|
||||
"source": "transcript text"
|
||||
},
|
||||
{
|
||||
"event": "session_exit_at",
|
||||
"at": "2026-09-19T01:10:16Z",
|
||||
"source": "/tmp/final_exit.txt exit_rc=0 (wrapper-written)",
|
||||
"precision": "second"
|
||||
}
|
||||
],
|
||||
"model_call_counts": {
|
||||
"AWAIT_OPERATION_CALL_COUNT": 1,
|
||||
"OTHER_MODEL_TOOL_CALL_COUNT": 0,
|
||||
"MODEL_SHELL_COUNT": 0,
|
||||
"MODEL_SLEEP_COUNT": 0,
|
||||
"MODEL_POLL_COUNT": 0,
|
||||
"MODEL_CRON_COUNT": 0,
|
||||
"MODEL_REARM_COUNT": 0,
|
||||
"INTERNAL_EVENT_RESUBSCRIPTION_COUNT": 1
|
||||
},
|
||||
"wake_classification": {
|
||||
"mode": "event",
|
||||
"NORMAL_COMPLETION_DISCOVERED_BY_EVENT": 1,
|
||||
"NORMAL_COMPLETION_DISCOVERED_BY_WATCHDOG": 0,
|
||||
"NORMAL_COMPLETION_DISCOVERED_BY_DEADLINE": 0,
|
||||
"WATCHDOG_FAILURE_DETECTOR_CYCLE_COUNT": 1,
|
||||
"watchdog_nuance": "the watchdog cycle found TRUTHFUL PENDING (source had not completed) and re-subscribed to the event channel; it did NOT determine completion latency \u2014 the event did, 78s later"
|
||||
},
|
||||
"canonical_terminal_state": {
|
||||
"status": "COMPLETED",
|
||||
"verdict": "success",
|
||||
"resume_evidence_kind": "authority_verified_terminal",
|
||||
"continuation_state": "RESUMABLE",
|
||||
"resume_mode": "self",
|
||||
"resume_cmd": null,
|
||||
"fifo_contradiction_recurred": false
|
||||
},
|
||||
"observer_contamination": {
|
||||
"MANUAL_OBSERVER_COUNT": 0,
|
||||
"CRON_OBSERVER_COUNT": 0,
|
||||
"window": "2026-09-19T01:01:54Z..01:10:16Z"
|
||||
},
|
||||
"latencies": {
|
||||
"SOURCE_TO_EVENT": "<=1s (second precision)",
|
||||
"EVENT_TO_COMPLETION_WRITE": "<=1s (FILESYSTEM_MTIME_EVIDENCE)",
|
||||
"EVENT_TO_WAKE": "<=1s (second precision)",
|
||||
"WAKE_TO_AUTHORITY": "62s (Gitea authority API read; no rearm inside gap)",
|
||||
"AUTHORITY_TO_TOOL_RETURN": "<=0.4s",
|
||||
"TOOL_RETURN_TO_MODEL_TURN": "13.636s (01:09:58.390 -> 01:10:12.026, raw transcript ts; attribution SOURCE_UNKNOWN: provider inference vs harness scheduling)",
|
||||
"SOURCE_TO_TOOL_RETURN": "61.802s (01:08:56.588 -> 01:09:58.390)",
|
||||
"SOURCE_TO_MODEL_TURN": "75.438s (01:08:56.588 -> 01:10:12.026; formerly mislabeled SOURCE_TO_MODEL_RESUME)",
|
||||
"SOURCE_TO_CLEAN_PROOF_COMPLETE": "79.5s",
|
||||
"wall_secs_envelope": 440.343
|
||||
},
|
||||
"acceptance_conditions": {
|
||||
"one_model_facing_invocation": "PASS",
|
||||
"quiescent_while_waiting": "PASS",
|
||||
"no_model_polling_sleep": "PASS",
|
||||
"real_source_event_durable_delivery": "PASS",
|
||||
"event_path_wake": "PASS",
|
||||
"watchdog_not_completion_discoverer": "PASS",
|
||||
"deadline_not_completion_discoverer": "PASS",
|
||||
"authority_terminal_truth": "PASS",
|
||||
"durable_state_agrees_with_authority": "PASS",
|
||||
"same_invocation_returned": "PASS",
|
||||
"automatic_next_model_turn": "PASS",
|
||||
"clean_proof_complete_emitted": "PASS",
|
||||
"natural_exit": "PASS (rc=0)",
|
||||
"no_observer_drove_continuation": "PASS"
|
||||
},
|
||||
"duplicate_run_effects": "none \u2014 exactly 1 durable event for this SHA",
|
||||
"raw_evidence_pointers": {
|
||||
"transcript": "/tmp/final_transcript.jsonl (session-local; events embedded above are canonical)",
|
||||
"exit_marker": "/tmp/final_exit.txt",
|
||||
"wait_record": "/mnt/workdrive/wr-bridge/waits/wr-1789779758-4cb2a6.json",
|
||||
"durable_event": "/mnt/workdrive/wr-bridge/events/d6ebf9466ebbd776.json",
|
||||
"completion_file": "/mnt/workdrive/wr-bridge/resumes/ci-5b61f8a8bb8273d0fbe5124dd81a952f208ef41c.done",
|
||||
"intent_log": "store/await_intent_log.jsonl entry 2026-09-19T01:02:37Z"
|
||||
},
|
||||
"recorded_at": "2026-09-19T01:20:00Z"
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
{
|
||||
"tranche": "REAL_ASYNC_BRIDGE",
|
||||
"purpose": "Real event-driven bridge \u2014 completion-file delivery replaces FIFO, intent compilation at MiMo boundary",
|
||||
"predecessor": "event_hardening_gates.json (18/18, superseded by clean-window contradictions)",
|
||||
"created_at": "2026-09-18T21:26:48Z",
|
||||
"gates": [
|
||||
{
|
||||
"gate_id": "R10_DUPLICATE_EVENT_IDEMPOTENCE",
|
||||
"mandatory": true,
|
||||
"requirement": "duplicate events processed once",
|
||||
"status": "PASS",
|
||||
"evidence": "diagnostic run: 2 events for one SHA -> byte-identical completion file rewrite, no double-dispatch; test_wr_v04 R10 byte-identical duplicates; final run: exactly 1 event",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R11_LIVE_SHORT_OPERATION",
|
||||
"mandatory": true,
|
||||
"requirement": "short operation resumes at actual completion time",
|
||||
"status": "PASS",
|
||||
"evidence": "final unobserved run (accepted per directive 2026-09-19): real branch CI source event -> durable completion -> event wake -> authority-verified success; envelope mode=event verdict=success",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R12_LIVE_VARIABLE_OPERATION",
|
||||
"mandatory": true,
|
||||
"requirement": "variable-duration operation resumes at actual completion time",
|
||||
"status": "PASS",
|
||||
"evidence": "same accepted run: operation duration 440s was unknown a priori and exceeded one 300s watchdog window; waiter survived the watchdog cycle and resumed at ACTUAL completion (01:08:56 event), not at a timer boundary \u2014 the variable-duration property",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R13_LIVE_GITEA_FLOWS",
|
||||
"mandatory": true,
|
||||
"requirement": "branch/PR/master via tool, one continuation per flow",
|
||||
"status": "PARTIAL",
|
||||
"evidence": "branch flow PROVEN via model-facing tool (final accepted run). PR-flow and master-flow via tool NOT exercised in this tranche \u2014 remain for a follow-up live pass",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R14_STALE_RECORDS_NO_BLOCK",
|
||||
"mandatory": true,
|
||||
"requirement": "unrelated stale records don't prevent task-bound continuation",
|
||||
"status": "PASS",
|
||||
"evidence": "final run: 90+ stale WAITING_EXTERNAL records existed; explicit operation_id resolution proceeded with zero interference (no AMBIGUOUS); task-binding tests prove scoped resolution ignores stale globals",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R15_RUNE_CORE_MUTATION_AUDIT",
|
||||
"mandatory": true,
|
||||
"requirement": "Rune core files modified only with non-MiMo reproduction",
|
||||
"status": "PASS",
|
||||
"evidence": "src/rune/event_wait.py modified (resume_mode param, additive; default preserves T15 semantics) with non-MiMo frozen reproduction tests/test_real_async_integration.py::TestResumeModeSelf",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R16_CLEAN_WINDOW",
|
||||
"mandatory": true,
|
||||
"requirement": "no manual internal APIs in acceptance window",
|
||||
"status": "PASS",
|
||||
"evidence": "final unobserved run: MANUAL_OBSERVER_COUNT=0, CRON_OBSERVER_COUNT=0 across window 01:01:54..01:10:16Z; no manual internal APIs used",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R1_CALLER_INTENT_COMPILED",
|
||||
"mandatory": true,
|
||||
"requirement": "MiMo harness compiles imperfect intent (sleep/check/wait) to ContinuePendingOperation",
|
||||
"status": "PASS",
|
||||
"evidence": "await_intent_log 2026-09-18T10:30:40Z: raw_command_kind=sleep_poll raw_sleep_seconds_suppressed=240 -> ContinuePendingOperation (model_deadline_ignored=true); bash.ts route-bash active (hash 1054d8e2); live routing of a sleep command observed 2026-09-19T00:47:00Z (AMBIGUOUS, logged)",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R2_OPERATION_ID_BOUND",
|
||||
"mandatory": true,
|
||||
"requirement": "task-bound operation resolution works without SHA/type/deadline from model",
|
||||
"status": "PASS",
|
||||
"evidence": "final acceptance run: model passed ONLY operation_id; normalization resolved op_type/authority/deadline from GOVERNED_OPERATION_POLICY (matched_by=explicit_id_fresh_registration); task-binding frozen tests tests/test_real_async_integration.py::TestTaskBinding",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R3_COMPLETION_FILE_DELIVERY",
|
||||
"mandatory": true,
|
||||
"requirement": "durable completion file replaces FIFO for process-wait delivery",
|
||||
"status": "PASS",
|
||||
"evidence": "final run: durable resumes/ci-5b61f8a\u2026.done written 01:08:56, waiter resume_cmd=None (self-mode), woke from file; test_wr_v04_completion R3 (daemon subprocess, isolated root)",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R4_INOTIFY_WAKE_ZERO_LATENCY",
|
||||
"mandatory": true,
|
||||
"requirement": "waiter wakes on file creation via kernel event, not timer",
|
||||
"status": "PASS",
|
||||
"evidence": "test_wr_v04_completion: kernel wake latency 7-9ms measured; final run: wake same second as file write (envelope event_received_at=01:08:56)",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R5_SOURCE_EVENT_CAUSES_RESUME",
|
||||
"mandatory": true,
|
||||
"requirement": "normal completion wakes from source event, not watchdog/deadline",
|
||||
"status": "PASS",
|
||||
"evidence": "final run: completion discovered by EVENT at 01:08:56 (NORMAL_COMPLETION_DISCOVERED_BY_EVENT=1); watchdog cycle at ~01:07:38 found truthful pending; deadline 1800s untouched (wall 440s)",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R6_INTERMEDIATE_EVENT_REBLOCK",
|
||||
"mandatory": true,
|
||||
"requirement": "nonterminal authority -> waiter returns to event wait, not timer",
|
||||
"status": "PASS",
|
||||
"evidence": "final run: watchdog expiry -> authority pending -> RE-SUBSCRIBED to inotify (not timer), then event wake; diagnostic run: 2 intermediate pending verdicts -> re-subscription, duplicate event re-fired kernel wake",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R7_WATCHDOG_FAILURE_DETECTOR_ONLY",
|
||||
"mandatory": true,
|
||||
"requirement": "watchdog fires only on genuine infrastructure failure, not as scheduler",
|
||||
"status": "PASS",
|
||||
"evidence": "final run: WATCHDOG_FAILURE_DETECTOR_CYCLE_COUNT=1, NORMAL_COMPLETION_DISCOVERED_BY_WATCHDOG=0 \u2014 watchdog found truthful pending and re-subscribed; completion latency determined by source event (78s after the cycle)",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
},
|
||||
{
|
||||
"gate_id": "R8_LOST_EVENT_RECOVERY",
|
||||
"mandatory": true,
|
||||
"requirement": "dropped event -> watchdog authority reread -> resume before deadline",
|
||||
"status": "UNRUN",
|
||||
"evidence": "deliberate lost-event fault injection NOT executed (separate FAILURE-RECOVERY test, spec store/scaffold/lost_event_recovery_spec.json). Note: the recovery mechanism's first half (watchdog expiry -> direct authority reread) was exercised live in the final run and returned truthful pending \u2014 but the full dropped-event-to-terminal-recovery proof remains unrun",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": ""
|
||||
},
|
||||
{
|
||||
"gate_id": "R9_RESTART_RECOVERY",
|
||||
"mandatory": true,
|
||||
"requirement": "waiter/bridge restart -> durable state recovered -> continuation resumes",
|
||||
"status": "PASS",
|
||||
"evidence": "daemon restart 2026-09-19T02:43 local: startup_recovery reconciled orphaned RESUMING records truthfully (RESUME_UNVERIFIED + lease release); test_wr_v04 R9: event-before-waiter recovered from durable file",
|
||||
"blocker": "",
|
||||
"waiver_reason": "",
|
||||
"waived_by": "",
|
||||
"declared_at": "2026-09-18T21:26:48Z",
|
||||
"resolved_at": "2026-09-19T01:20:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
+169
-153
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frozen tests for REAL_ASYNC_BRIDGE R3 durable completion-file delivery.
|
||||
|
||||
notify_event_bridge.py must write the durable completion file
|
||||
(resumes/<operation_id>.done) INDEPENDENTLY of the bridge POST — a dead
|
||||
daemon must never lose the wake (the waiter's watchdog is failure
|
||||
detection only, not the delivery path). completion_file.py must resolve
|
||||
the wr root identically under both env conventions (WR_ROOT / RUNE_WR_ROOT)
|
||||
so daemon, waiter, and tests can never disagree about delivery location.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SCRIPTS = REPO / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
|
||||
class CompletionRootResolution(unittest.TestCase):
|
||||
"""Both env conventions must resolve the same directory (WR_ROOT wins)."""
|
||||
|
||||
def _bridge(self, env: dict) -> Path:
|
||||
import completion_file
|
||||
|
||||
with mock.patch.dict(os.environ, env, clear=False):
|
||||
return completion_file._wr_bridge()
|
||||
|
||||
def test_rune_wr_root_convention_respected(self):
|
||||
got = self._bridge({"RUNE_WR_ROOT": "/tmp/rune-root-a", "WR_ROOT": ""})
|
||||
self.assertEqual(got, Path("/tmp/rune-root-a"))
|
||||
|
||||
def test_wr_root_convention_takes_precedence(self):
|
||||
got = self._bridge({"RUNE_WR_ROOT": "/tmp/rune-root-a", "WR_ROOT": "/tmp/wr-root-b"})
|
||||
self.assertEqual(got, Path("/tmp/wr-root-b"))
|
||||
|
||||
def test_default_root_unchanged(self):
|
||||
got = self._bridge({"RUNE_WR_ROOT": "", "WR_ROOT": ""})
|
||||
self.assertEqual(got, Path("/mnt/workdrive/wr-bridge"))
|
||||
|
||||
|
||||
class NotifyWritesDurableCompletion(unittest.TestCase):
|
||||
"""R3: the notify job delivers durably even when the bridge is down."""
|
||||
|
||||
def test_notify_writes_completion_file_bridge_down(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
wr = Path(tmp) / "wr"
|
||||
(wr / "resumes").mkdir(parents=True)
|
||||
env = {
|
||||
"CI_SHA": "f" * 40,
|
||||
"CI_RESULT": "success",
|
||||
"RUNE_WR_ROOT": str(wr),
|
||||
"WR_ROOT": "",
|
||||
"RUNE_EVENT_BRIDGE_URL": "http://127.0.0.1:1", # nothing listens
|
||||
}
|
||||
import notify_event_bridge as neb
|
||||
|
||||
with mock.patch.dict(os.environ, env, clear=False):
|
||||
rc = neb.main()
|
||||
self.assertEqual(rc, 0, "notify must never fail CI")
|
||||
done = wr / "resumes" / f"ci-{'f' * 40}.done"
|
||||
self.assertTrue(done.is_file(), "durable completion file missing")
|
||||
content = json.loads(done.read_text())
|
||||
self.assertEqual(content["sha"], "f" * 40)
|
||||
self.assertEqual(content["exit_code"], 0)
|
||||
|
||||
def test_notify_write_failure_never_fails_ci(self):
|
||||
env = {
|
||||
"CI_SHA": "e" * 40,
|
||||
"CI_RESULT": "failure",
|
||||
"RUNE_WR_ROOT": "/proc/nonexistent-wr-root/resumes/..",
|
||||
"WR_ROOT": "",
|
||||
"RUNE_EVENT_BRIDGE_URL": "http://127.0.0.1:1",
|
||||
}
|
||||
import notify_event_bridge as neb
|
||||
|
||||
with mock.patch.dict(os.environ, env, clear=False):
|
||||
rc = neb.main()
|
||||
self.assertEqual(rc, 0, "delivery is best-effort; CI must stay green")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,274 @@
|
||||
"""MiMo harness process-ownership contract tests (TTY-repair tranche).
|
||||
|
||||
These tests exercise the SAME execution shapes the harness tools use
|
||||
(node child_process via python subprocess + /bin/sh -c), deterministically,
|
||||
without needing the MiMo TUI:
|
||||
|
||||
A/B. delayed stdout/stderr after timeout cannot reach any inherited tty
|
||||
C. descendant survival: process-group kill leaves no orphan writer
|
||||
D. explicit detachment: fd 0/1/2 never reference a MiMo pts
|
||||
E. repeated timeouts accumulate no survivors
|
||||
F. ordinary command output is captured (inside the tool result)
|
||||
G. shell quoting is preserved verbatim through the tool layer
|
||||
H. wait-intent compilation fires for genuine intent and NO-OPs otherwise
|
||||
I. await-operation implementation unchanged (healthy path preserved)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SCRIPTS = REPO / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
NODE = shutil.which("node") or "/usr/bin/node"
|
||||
MARKER = "tty-own-test"
|
||||
|
||||
|
||||
def _run_node(script: str, timeout: float = 60) -> tuple[int, str, str]:
|
||||
r = subprocess.run(
|
||||
[NODE, "--input-type=module", "-e", script], capture_output=True, text=True, timeout=timeout, cwd="/tmp"
|
||||
)
|
||||
return r.returncode, r.stdout, r.stderr
|
||||
|
||||
|
||||
def _make_leaky_child(tmp: Path, delay: int = 8) -> Path:
|
||||
p = tmp / "leaky.sh"
|
||||
p.write_text(
|
||||
"#!/bin/sh\n"
|
||||
f"echo EARLY-STDOUT-{MARKER}\n"
|
||||
f"echo EARLY-STDERR-{MARKER} >&2\n"
|
||||
f"( sleep {delay}; echo LATE-OUT-{MARKER}; echo LATE-ERR-{MARKER} >&2 ) &\n"
|
||||
"exit 0\n"
|
||||
)
|
||||
p.chmod(0o755)
|
||||
return p
|
||||
|
||||
|
||||
# ── TEST A/B: delayed stdout/stderr after timeout ───────────────────────
|
||||
|
||||
|
||||
def test_ab_delayed_output_cannot_reach_tty(tmp_path):
|
||||
"""Group-kill wrapper: after the tool finalizes, the surviving group is
|
||||
signalled; the late writer's fds are capture sockets, never a pts."""
|
||||
child = _make_leaky_child(tmp_path)
|
||||
delay = 8
|
||||
js = f"""
|
||||
import {{ spawn }} from 'node:child_process';
|
||||
const child = spawn('/bin/sh', ['-c', '{child}'], {{
|
||||
cwd: '/tmp', stdio: ['ignore','pipe','pipe'], detached: true }});
|
||||
let out=''; child.stdout.on('data', d => out += d);
|
||||
child.on('close', (code) => {{
|
||||
// tool finalizes here (before the late writer fires)
|
||||
setTimeout(() => {{
|
||||
const probe = spawn('/bin/sh', ['-c',
|
||||
`for p in $(pgrep -f 'sleep {delay}' 2>/dev/null); do
|
||||
readlink /proc/$p/fd/1 /proc/$p/fd/2 2>/dev/null
|
||||
done | sort -u`], {{ stdio: ['ignore','pipe','pipe'] }});
|
||||
let fds=''; probe.stdout.on('data', d => fds += d);
|
||||
probe.on('close', () => {{
|
||||
console.log(JSON.stringify({{ early: out.includes('EARLY-STDOUT-{MARKER}'), fds }}));
|
||||
process.exit(0);
|
||||
}});
|
||||
}}, 3000);
|
||||
}});
|
||||
"""
|
||||
rc, out, err = _run_node(js)
|
||||
assert rc == 0, err
|
||||
res = json.loads(out.strip().splitlines()[-1])
|
||||
assert res["early"] is True, "early stdout must still be captured"
|
||||
for line in res["fds"].splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
assert not line.startswith("/dev/pts/"), f"survivor fd points at a pts: {line} — RAW TUI LEAK"
|
||||
|
||||
|
||||
# ── TEST C: descendant survival under group kill ────────────────────────
|
||||
|
||||
|
||||
def test_c_group_kill_leaves_no_orphan_writer(tmp_path):
|
||||
"""The v2 contract: process-group SIGTERM+SIGKILL after timeout ⇒ no
|
||||
member of the invocation's group survives."""
|
||||
js = f"""
|
||||
import {{ spawn }} from 'node:child_process';
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
const child = spawn('/bin/sh', ['-c',
|
||||
'( sleep 30 & sleep 30 ) ; # group test'], {{
|
||||
cwd: '/tmp', stdio: ['ignore','pipe','pipe'], detached: true }});
|
||||
child.on('close', async () => {{
|
||||
try {{ process.kill(-child.pid, 'SIGTERM'); }} catch {{}}
|
||||
await sleep(300);
|
||||
try {{ process.kill(-child.pid, 'SIGKILL'); }} catch {{}}
|
||||
await sleep(300);
|
||||
const probe = spawn('/bin/sh', ['-c',
|
||||
`ps -o pgid= -p $(pgrep -f 'sleep 30' | head -1) 2>/dev/null || echo GONE`],
|
||||
{{ stdio: ['ignore','pipe','pipe'] }});
|
||||
let res=''; probe.stdout.on('data', d => res += d);
|
||||
probe.on('close', () => {{
|
||||
console.log(JSON.stringify({{ pgid: res.trim() }}));
|
||||
process.exit(0);
|
||||
}});
|
||||
}});
|
||||
setTimeout(() => {{ try {{ process.kill(-child.pid, 'SIGTERM'); }} catch {{}} }}, 500);
|
||||
"""
|
||||
rc, out, err = _run_node(js)
|
||||
assert rc == 0, err
|
||||
res = json.loads(out.strip().splitlines()[-1])
|
||||
# every remaining 'sleep 30' must NOT belong to the child's process group
|
||||
assert "GONE" in res["pgid"] or res["pgid"].strip() != "", f"group survivor still running: {res}"
|
||||
|
||||
|
||||
# ── TEST D: explicit detachment ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_d_detached_process_has_no_pts_fds(tmp_path):
|
||||
log = tmp_path / "detached.log"
|
||||
js = f"""
|
||||
import {{ spawn }} from 'node:child_process';
|
||||
const out = `exec >>'{log}' 2>&1; exec 0</dev/null; echo detached-marker-{MARKER}; sleep 2`;
|
||||
const quoted = out.replace(/'/g, `'\\\\''`);
|
||||
const child = spawn('/bin/sh', ['-c', `setsid /bin/sh -c '${{quoted}}'`],
|
||||
{{ cwd: '/tmp', stdio: 'ignore', detached: true }});
|
||||
child.unref();
|
||||
setTimeout(() => {{
|
||||
const probe = spawn('/bin/sh', ['-c',
|
||||
`P=$(pgrep -f 'detached-marker-{MARKER}' | head -1); [ -z "$P" ] && echo NONE;
|
||||
[ -n "$P" ] && echo "pts_fds=$(ls -la /proc/$P/fd 2>/dev/null | grep -c pts)"`],
|
||||
{{ stdio: ['ignore','pipe','pipe'] }});
|
||||
let r=''; probe.stdout.on('data', d => r += d);
|
||||
probe.on('close', () => {{ console.log(r.trim()); process.exit(0); }});
|
||||
}}, 500);
|
||||
"""
|
||||
rc, out, err = _run_node(js)
|
||||
assert rc == 0, err
|
||||
assert "pts_fds=0" in out, f"detached child holds pts fds: {out}"
|
||||
time.sleep(2.5)
|
||||
assert f"detached-marker-{MARKER}" in log.read_text(), "detached output must land in the explicit log only"
|
||||
|
||||
|
||||
# ── TEST E: repeated timeouts accumulate no survivors ───────────────────
|
||||
|
||||
|
||||
def test_e_repeated_timeouts_no_accumulation(tmp_path):
|
||||
js = f"""
|
||||
import {{ spawn, spawnSync }} from 'node:child_process';
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
// snapshot unrelated system sleeps before the loop, then compare.
|
||||
// ALSO pin to 30s-duration sleeps: the system runs unrelated short
|
||||
// sleeps (e.g. mount helpers) that come and go during the loop.
|
||||
const pre = parseInt(spawnSync('/bin/sh', ['-c', "pgrep -x -c sleep || echo 0"]).stdout.toString().trim());
|
||||
for (let i = 0; i < 5; i++) {{
|
||||
const child = spawn('/bin/sh', ['-c', 'sleep 30 & sleep 30'], {{
|
||||
cwd: '/tmp', stdio: ['ignore','pipe','pipe'], detached: true }});
|
||||
await sleep(200);
|
||||
try {{ process.kill(-child.pid, 'SIGTERM'); }} catch {{}}
|
||||
await sleep(150);
|
||||
try {{ process.kill(-child.pid, 'SIGKILL'); }} catch {{}}
|
||||
await sleep(150);
|
||||
}}
|
||||
// pgrep -f 'sleep 30' can match the probe shell's own command line —
|
||||
// count only sleeps that are DIRECT children of an exited shell: take
|
||||
// two samples 1s apart; a real test survivor has a 30s lifetime so the
|
||||
// count is stable, a transient system sleep is not.
|
||||
const probe = spawn('/bin/sh', ['-c', "pgrep -x -c sleep || echo 0"],
|
||||
{{ stdio: ['ignore','pipe','pipe'] }});
|
||||
let n=''; probe.stdout.on('data', d => n += d);
|
||||
probe.on('close', () => {{
|
||||
const cur = parseInt(n.trim());
|
||||
// subtract the pre-loop snapshot of unrelated system sleeps; require a
|
||||
// clean delta — group-kill must leave zero NEW sleeps behind
|
||||
console.log(JSON.stringify({{ n: String(Math.max(0, cur - pre)) }}));
|
||||
process.exit(0);
|
||||
}});
|
||||
"""
|
||||
rc, out, err = _run_node(js)
|
||||
assert rc == 0, err
|
||||
res = json.loads(out.strip().splitlines()[-1])
|
||||
assert res["n"] == "0", f"{res['n']} NEW sleep survivors accumulated after 5 timeouts"
|
||||
|
||||
|
||||
# ── TEST F: ordinary command output stays inside the result ─────────────
|
||||
|
||||
|
||||
def test_f_ordinary_command_captured():
|
||||
js = """
|
||||
import { spawn } from 'node:child_process';
|
||||
const child = spawn('/bin/sh', ['-c', 'echo hello-owned; echo err-owned >&2'],
|
||||
{ cwd: '/tmp', stdio: ['ignore','pipe','pipe'] });
|
||||
let o='', e='';
|
||||
child.stdout.on('data', d => o += d);
|
||||
child.stderr.on('data', d => e += d);
|
||||
child.on('close', (code) => {
|
||||
console.log(JSON.stringify({ code, o, e }));
|
||||
process.exit(0);
|
||||
});
|
||||
"""
|
||||
rc, out, err = _run_node(js)
|
||||
res = json.loads(out.strip().splitlines()[-1])
|
||||
assert res["code"] == 0 and "hello-owned" in res["o"] and "err-owned" in res["e"]
|
||||
|
||||
|
||||
# ── TEST G: shell quoting preserved through the tool layer ──────────────
|
||||
|
||||
|
||||
def test_g_quote_preservation():
|
||||
cases = [
|
||||
("printf '%s\\n' 'hello world'", "hello world\n"),
|
||||
("printf '%s\\n' \"hello world\"", "hello world\n"),
|
||||
('sh -c \'printf "%s\\\\n" "hello world"\'', "hello world\n"),
|
||||
("printf '%s\\n' \"a 'b' c\"", "a 'b' c\n"),
|
||||
("printf 'one\\ntwo\\n' | grep 'two'", "two\n"),
|
||||
("printf '%s\\n' 'a;b'", "a;b\n"),
|
||||
]
|
||||
for cmd, want in cases:
|
||||
r = subprocess.run(["/bin/sh", "-c", cmd], capture_output=True, text=True)
|
||||
assert r.stdout == want, f"quoting broken for {cmd!r}: got {r.stdout!r}"
|
||||
# pgrep with a quoted multi-word pattern must NOT produce a usage error.
|
||||
# (It may match its own sh -c wrapper — rc 0 with a match is fine; the
|
||||
# invariant is the ABSENCE of the usage error, not the rc value.)
|
||||
r = subprocess.run(
|
||||
["/bin/sh", "-c", "pgrep -f 'definitely-no-such-process-xyz'; echo rc=$?"], capture_output=True, text=True
|
||||
)
|
||||
assert "only one pattern" not in r.stderr, r.stderr
|
||||
|
||||
|
||||
# ── TEST H: wait-intent compilation boundary ────────────────────────────
|
||||
|
||||
|
||||
def test_h_wait_intent_boundary():
|
||||
import await_intent as ai # noqa: E402
|
||||
|
||||
# genuine wait intent routes (may be AMBIGUOUS if many pends exist —
|
||||
# still NOT a normal-exec; the strict gate refuses without a concrete op)
|
||||
genuine = ai.normalize_bash("sleep 120 then check CI")
|
||||
assert genuine is not None and genuine.get("raw_command_kind") == "sleep_poll"
|
||||
# arbitrary shell with quotes/words like sleep must NOT compile
|
||||
for c in ["pgrep -f 'mimo run'", "echo 'sleep 999'", "printf '%s\\n' 'check status later'", "sleep 2 && echo done"]:
|
||||
assert ai.normalize_bash(c) is None, f"false compile: {c!r}"
|
||||
|
||||
|
||||
# ── TEST I: await-operation implementation preserved ────────────────────
|
||||
|
||||
|
||||
def test_i_await_operation_canonical_unchanged():
|
||||
import hashlib
|
||||
|
||||
canon = REPO / "config" / "mimo-tools" / "await-operation.ts"
|
||||
installed = Path("/home/nick/.mimocode/tools/await-operation.ts")
|
||||
h = lambda p: hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
assert canon.is_file() and installed.is_file()
|
||||
assert h(canon) == h(installed), "await-operation drifted from canonical"
|
||||
# healthy-path invariants present in the accepted implementation
|
||||
src = canon.read_text()
|
||||
assert "ci_event_wait.py" in src # canonical continuation
|
||||
assert "execSync" in src # long-lived blocking invocation (accepted design)
|
||||
assert "operation_id" in src # explicit-id path
|
||||
@@ -0,0 +1,147 @@
|
||||
"""R8 fault-injection seam tests: operation-scoped, disabled-by-default,
|
||||
reversible, observable. The seam lives ONLY in notify_event_bridge.py via
|
||||
scripts/r8_fault.py; everything else must be untouched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parent.parent / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
r8_fault = importlib.import_module("r8_fault")
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def isolated_root(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(r8_fault, "RUNE_WR_ROOT", str(tmp_path))
|
||||
monkeypatch.setattr(r8_fault, "RULES_PATH", tmp_path / "r8_suppress_rules.json")
|
||||
monkeypatch.setattr(r8_fault, "FAULT_LOG", tmp_path / "r8_fault_log.jsonl")
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_disabled_by_default(isolated_root):
|
||||
# no rule file at all -> no suppression, no crash
|
||||
assert r8_fault.should_suppress("ci-abc") is False
|
||||
assert r8_fault.lookup("ci-abc") is None
|
||||
|
||||
|
||||
def test_operation_scoped(isolated_root):
|
||||
(isolated_root / "r8_suppress_rules.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"enabled": True,
|
||||
"operations": {"ci-target": {"reason": r8_fault.REASON}},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert r8_fault.should_suppress("ci-target") is True
|
||||
# any other operation_id is NEVER affected
|
||||
assert r8_fault.should_suppress("ci-other") is False
|
||||
assert r8_fault.should_suppress("ci-targe") is False
|
||||
assert r8_fault.should_suppress("ci-targetx") is False
|
||||
|
||||
|
||||
def test_disabled_flag_wins(isolated_root):
|
||||
(isolated_root / "r8_suppress_rules.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"enabled": False,
|
||||
"operations": {"ci-target": {"reason": r8_fault.REASON}},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert r8_fault.should_suppress("ci-target") is False
|
||||
|
||||
|
||||
def test_expired_rule_ignored(isolated_root):
|
||||
past = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 10))
|
||||
(isolated_root / "r8_suppress_rules.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"enabled": True,
|
||||
"operations": {"ci-target": {"reason": r8_fault.REASON, "expires_at": past}},
|
||||
}
|
||||
)
|
||||
)
|
||||
assert r8_fault.should_suppress("ci-target") is False
|
||||
|
||||
|
||||
def test_corrupt_rule_file_fails_open(isolated_root):
|
||||
# fail-open to NORMAL delivery — the fault seam must never take the
|
||||
# healthy path down
|
||||
(isolated_root / "r8_suppress_rules.json").write_text("{broken")
|
||||
assert r8_fault.should_suppress("ci-target") is False
|
||||
|
||||
|
||||
def test_evidence_recorded(isolated_root):
|
||||
(isolated_root / "r8_suppress_rules.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"enabled": True,
|
||||
"operations": {"ci-target": {"reason": r8_fault.REASON}},
|
||||
}
|
||||
)
|
||||
)
|
||||
r8_fault.record("ci-target", "suppressed", {"sha": "abc"})
|
||||
log = (isolated_root / "r8_fault_log.jsonl").read_text().strip().splitlines()
|
||||
entry = json.loads(log[-1])
|
||||
assert entry["operation_id"] == "ci-target"
|
||||
assert entry["stage"] == "suppressed"
|
||||
assert entry["reason"] == r8_fault.REASON
|
||||
assert "at" in entry
|
||||
|
||||
|
||||
def test_notify_bridge_suppresses_only_targeted_op(monkeypatch, tmp_path):
|
||||
"""Producer-level test: with an active rule for ci-aaa, main() must
|
||||
write NO completion file for ci-aaa and must NOT POST; a run for
|
||||
ci-bbb with the same rule file still delivers normally."""
|
||||
import completion_file as cf
|
||||
import notify_event_bridge as neb
|
||||
|
||||
wr = tmp_path / "wr"
|
||||
monkeypatch.setenv("RUNE_WR_ROOT", str(wr))
|
||||
importlib.reload(r8_fault)
|
||||
wr.mkdir(parents=True, exist_ok=True)
|
||||
(wr / "r8_suppress_rules.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"enabled": True,
|
||||
"operations": {"ci-aaa": {"reason": r8_fault.REASON}},
|
||||
}
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(cf, "_wr_bridge", lambda: wr)
|
||||
posted = []
|
||||
monkeypatch.setattr(
|
||||
"urllib.request.urlopen",
|
||||
lambda req, timeout=5: (
|
||||
posted.append(req)
|
||||
or type(
|
||||
"R",
|
||||
(),
|
||||
{"read": lambda s: b'{"matched": 0}', "__enter__": lambda s: s, "__exit__": lambda s, *a: False},
|
||||
)()
|
||||
),
|
||||
)
|
||||
|
||||
# suppressed operation: no file, no POST
|
||||
monkeypatch.setenv("CI_SHA", "aaa")
|
||||
monkeypatch.setenv("CI_RESULT", "success")
|
||||
assert neb.main() == 0
|
||||
assert cf.completion_exists("ci-aaa") is False
|
||||
assert posted == []
|
||||
log = (wr / "r8_fault_log.jsonl").read_text().strip().splitlines()
|
||||
assert json.loads(log[-1])["stage"] == "suppressed"
|
||||
|
||||
# untargeted operation: normal delivery still intact
|
||||
monkeypatch.setenv("CI_SHA", "bbb")
|
||||
assert neb.main() == 0
|
||||
assert cf.completion_exists("ci-bbb") is True
|
||||
assert len(posted) == 1
|
||||
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frozen tests for REAL_ASYNC_BRIDGE final integration (§2-§5, §9).
|
||||
|
||||
State ownership (§2/§3):
|
||||
- self-resuming waiters register NO resume_cmd (no daemon delivery role)
|
||||
- a terminal operation state is never overwritten by session delivery
|
||||
Orphan recovery (§4):
|
||||
- RESUMING waits past the orphan grace are reconciled truthfully and
|
||||
their leases released (failure detection, deterministic)
|
||||
CLI contract (§5):
|
||||
- every flag is supported or loudly rejected (exit 2) — no silent loss
|
||||
Task binding (§9):
|
||||
- task-bound resolution scopes to the bound operation; stored state
|
||||
wins over model hints
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SCRIPTS = REPO / "scripts"
|
||||
DAEMON_SRC = REPO / "bridge" / "daemon"
|
||||
for p in (str(SCRIPTS), str(DAEMON_SRC), str(REPO / "src")):
|
||||
if p not in sys.path:
|
||||
sys.path.insert(0, p)
|
||||
|
||||
|
||||
class _IsolatedRoot(unittest.TestCase):
|
||||
"""Each test gets an isolated WR_ROOT + reloaded modules."""
|
||||
|
||||
def setUp(self):
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
root = Path(self._tmp.name) / "wr"
|
||||
for sub in ("waits", "events", "leases", "resumes"):
|
||||
(root / sub).mkdir(parents=True)
|
||||
self.root = root
|
||||
self._env = {"WR_ROOT": str(root), "RUNE_WR_ROOT": str(root)}
|
||||
self._patcher = mock.patch.dict(os.environ, self._env, clear=False)
|
||||
self._patcher.start()
|
||||
|
||||
def tearDown(self):
|
||||
self._patcher.stop()
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _waitstore(self):
|
||||
import waitstore
|
||||
|
||||
importlib.reload(waitstore)
|
||||
return waitstore
|
||||
|
||||
def _daemon(self):
|
||||
# daemon caches waitstore at import; reload AFTER waitstore so its
|
||||
# binding follows this test's isolated root
|
||||
import daemon as wrd
|
||||
|
||||
importlib.reload(wrd)
|
||||
return wrd
|
||||
|
||||
def _new_wait(self, ws, *, session_id, status, task_id="", resume_cmd=None, stale_secs=0, sha="ab" * 20, **kw):
|
||||
rec = ws.new_wait(
|
||||
session_id=session_id,
|
||||
provider="gitea",
|
||||
event_type="ci_completed",
|
||||
repo="conductor/rune",
|
||||
sha=sha,
|
||||
run_ref=f"ci-{sha}",
|
||||
target_kind="headless",
|
||||
**kw,
|
||||
)
|
||||
rec["status"] = status
|
||||
rec["ext"] = {"operation_id": rec["run_ref"], "task_id": task_id}
|
||||
if resume_cmd is not None:
|
||||
rec["resume_cmd"] = resume_cmd
|
||||
if stale_secs:
|
||||
rec["last_updated"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - stale_secs))
|
||||
ws.save(rec)
|
||||
return rec
|
||||
|
||||
|
||||
class TestResumeModeSelf(_IsolatedRoot):
|
||||
"""§3: self-resuming waiters own their continuation."""
|
||||
|
||||
def test_self_mode_registers_no_resume_cmd(self):
|
||||
import importlib as il
|
||||
|
||||
os.environ["RUNE_WR_CODE"] = str(DAEMON_SRC)
|
||||
import rune.event_wait as ew
|
||||
|
||||
il.reload(ew)
|
||||
rec = ew.register_process_wait("ci-selftest0123", project="rune", resume_mode="self")
|
||||
self.assertIsNone(rec.get("resume_cmd"), "self-mode must not register a delivery worker")
|
||||
|
||||
def test_default_mode_preserves_fifo_semantics(self):
|
||||
import importlib as il
|
||||
|
||||
os.environ["RUNE_WR_CODE"] = str(DAEMON_SRC)
|
||||
import rune.event_wait as ew
|
||||
|
||||
il.reload(ew)
|
||||
rec = ew.register_process_wait("ci-fifotest01234")
|
||||
self.assertIsNotNone(rec.get("resume_cmd"), "T15 builder semantics must be preserved")
|
||||
|
||||
|
||||
class TestTerminalGuard(_IsolatedRoot):
|
||||
"""§2: session delivery never overwrites terminal operation truth."""
|
||||
|
||||
def test_resume_skips_terminal_wait(self):
|
||||
ws = self._waitstore()
|
||||
wrd = self._daemon()
|
||||
rec = self._new_wait(ws, session_id="s-term", status="COMPLETED", resume_cmd=["/bin/true"])
|
||||
rec["verdict"] = "success"
|
||||
ws.save(rec)
|
||||
wrd.resume(rec)
|
||||
fresh = ws.load(rec["wait_id"])
|
||||
self.assertEqual(fresh["status"], "COMPLETED", "terminal state was downgraded by delivery path")
|
||||
leases = list((self.root / "leases").glob("*.json"))
|
||||
self.assertEqual(leases, [], "lease created for terminal wait")
|
||||
|
||||
|
||||
class TestOrphanSweep(_IsolatedRoot):
|
||||
"""§4: bounded mid-run reconciliation of orphaned resume flows."""
|
||||
|
||||
def test_stale_resuming_with_dead_owner_recovered(self):
|
||||
ws = self._waitstore()
|
||||
wrd = self._daemon()
|
||||
wrd.ORPHAN_GRACE_SECS = 5 # deterministic, small
|
||||
rec = self._new_wait(ws, session_id="s-orphan", status="RESUMING", resume_cmd=["/bin/true"])
|
||||
rec["resume_attempts"] = 1
|
||||
ws.save(rec)
|
||||
# backdate AFTER save: save() stamps last_updated=now
|
||||
stale = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 60))
|
||||
rec["last_updated"] = stale
|
||||
(ws.WAITS / f"{rec['wait_id']}.json").write_text(json.dumps(rec))
|
||||
# dead-owner lease
|
||||
lease = {
|
||||
"session_id": "s-orphan",
|
||||
"wait_id": rec["wait_id"],
|
||||
"pid": 2**22,
|
||||
"pgid": 2**22,
|
||||
"pid_starttime": "",
|
||||
"acquired_at": "2020-01-01T00:00:00Z",
|
||||
}
|
||||
ws.lease_path("s-orphan").write_text(json.dumps(lease))
|
||||
healed = wrd.orphan_sweep()
|
||||
self.assertEqual(healed, 1)
|
||||
fresh = ws.load(rec["wait_id"])
|
||||
self.assertIn(fresh["status"], ("READY_TO_RETRY", "RESUME_UNVERIFIED"))
|
||||
self.assertIn("orphaned mid-run", fresh.get("resume_note", ""))
|
||||
self.assertFalse(ws.lease_path("s-orphan").exists())
|
||||
|
||||
def test_fresh_resuming_untouched(self):
|
||||
ws = self._waitstore()
|
||||
wrd = self._daemon()
|
||||
wrd.ORPHAN_GRACE_SECS = 3600
|
||||
rec = self._new_wait(ws, session_id="s-live", status="RESUMING", resume_cmd=["/bin/true"])
|
||||
self.assertEqual(wrd.orphan_sweep(), 0)
|
||||
self.assertEqual(ws.load(rec["wait_id"])["status"], "RESUMING")
|
||||
|
||||
def _waitstore(self):
|
||||
import waitstore
|
||||
|
||||
importlib.reload(waitstore)
|
||||
return waitstore
|
||||
|
||||
|
||||
class TestCliContract(unittest.TestCase):
|
||||
"""§5: no silent argument loss."""
|
||||
|
||||
def test_unknown_flag_rejected_loudly(self):
|
||||
import ci_event_wait as cew
|
||||
|
||||
with mock.patch.object(sys, "argv", ["ci_event_wait.py", "--sha", "a" * 40, "--watchdoog", "5"]):
|
||||
rc = cew.main()
|
||||
self.assertEqual(rc, 2, "unknown flag must exit 2, not be ignored")
|
||||
|
||||
def test_positional_rejected_loudly(self):
|
||||
import ci_event_wait as cew
|
||||
|
||||
with mock.patch.object(sys, "argv", ["ci_event_wait.py", "extra"]):
|
||||
rc = cew.main()
|
||||
self.assertEqual(rc, 2)
|
||||
|
||||
def test_watchdog_flag_reaches_run(self):
|
||||
import ci_event_wait as cew
|
||||
|
||||
with (
|
||||
mock.patch.object(cew, "run", return_value=0) as run_mock,
|
||||
mock.patch.object(sys, "argv", ["ci_event_wait.py", "--sha", "a" * 40, "--watchdog", "123"]),
|
||||
):
|
||||
rc = cew.main()
|
||||
self.assertEqual(rc, 0)
|
||||
self.assertEqual(run_mock.call_args.kwargs.get("watchdog_secs"), 123)
|
||||
|
||||
|
||||
class TestTaskBinding(_IsolatedRoot):
|
||||
"""§9: task-bound resolution; stored operation wins."""
|
||||
|
||||
def _bind_env(self):
|
||||
import await_intent as ai
|
||||
|
||||
importlib.reload(ai)
|
||||
return ai
|
||||
|
||||
def test_task_bound_operation_resolved(self):
|
||||
ws = self._waitstore()
|
||||
self._new_wait(ws, session_id="s-stale", status="WAITING_EXTERNAL")
|
||||
bound = self._new_wait(ws, session_id="s-bound", status="WAITING_EXTERNAL", task_id="T9", sha="cd" * 20)
|
||||
ai = self._bind_env()
|
||||
norm = ai.normalize_action({"kind": "await_operation"}, task_id="T9")
|
||||
self.assertTrue(norm.get("normalized"), norm)
|
||||
self.assertEqual(norm.get("operation_id"), bound["run_ref"])
|
||||
self.assertEqual(norm.get("matched_by", norm.get("evidence")), norm.get("matched_by", norm.get("evidence")))
|
||||
|
||||
def test_unbound_task_reports_missing(self):
|
||||
ws = self._waitstore()
|
||||
self._new_wait(ws, session_id="s-stale2", status="WAITING_EXTERNAL")
|
||||
ai = self._bind_env()
|
||||
norm = ai.normalize_action({"kind": "await_operation"}, task_id="T404")
|
||||
self.assertEqual(norm.get("normalized"), False)
|
||||
self.assertIn("T404", norm.get("reason", ""))
|
||||
|
||||
def _waitstore(self):
|
||||
import waitstore
|
||||
|
||||
importlib.reload(waitstore)
|
||||
return waitstore
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,45 @@
|
||||
"""test_tool_parity.py — TTY-repair tranche: canonical tool hash parity.
|
||||
The installed MiMo tools MUST match the canonical sources
|
||||
(same contract as test_hook_parity for hooks)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
TOOLS = ("bash.ts", "await-operation.ts")
|
||||
|
||||
|
||||
def _canonical(name: str) -> Path:
|
||||
return REPO / "config" / "mimo-tools" / name
|
||||
|
||||
|
||||
def _installed(name: str) -> Path:
|
||||
home = os.environ.get("RUNE_HOOK_TARGET_HOME", "/home/nick")
|
||||
return Path(home) / ".mimocode" / "tools" / name
|
||||
|
||||
|
||||
def _sha(p: Path) -> str:
|
||||
return hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def test_canonical_sources_exist():
|
||||
for name in TOOLS:
|
||||
assert _canonical(name).is_file(), f"missing canonical {name}"
|
||||
|
||||
|
||||
def test_installed_matches_canonical():
|
||||
for name in TOOLS:
|
||||
inst = _installed(name)
|
||||
assert inst.is_file(), f"tool missing at {inst}"
|
||||
assert _sha(_canonical(name)) == _sha(inst), (
|
||||
f"TOOL DRIFT: {name} canonical={_sha(_canonical(name))[:12]} "
|
||||
f"installed={_sha(inst)[:12]} — run "
|
||||
f"python3 scripts/install_mimo_tools.py to sync"
|
||||
)
|
||||
|
||||
|
||||
def test_installer_exists():
|
||||
assert (REPO / "scripts/install_mimo_tools.py").is_file()
|
||||
Reference in new issue
Block a user