ROOT CAUSE (red-CI promotions 0.66.28/29): the required-checks 'Publish protected-branch gate' step read /tmp/need_* files that no step ever wrote and echoed 'All required CI checks passed' UNCONDITIONALLY. With if: always() the needs gate is overridden, so the job ran on red needs and published success — deploy_qualify correctly trusted it and promoted red commits (24413e4,fd6e9b4, de30cd9; status timelines in the closure evidence). FIX: publish step now runs only when every needs.*.result == 'success'; companion fail-closed step exits 1 otherwise, so the required-checks context reports failure instead of lying. DEFENSE-IN-DEPTH: deploy_qualify verdict is now the AND of two independent legs — (1) newest required-checks observation for the exact SHA, (2) the latest COMPLETED ci.yml@refs/heads/master run for that exact SHA with every job concluded success (fan-out resolved by path, run resolved by head_sha, jobs envelope unwrapped — no aggregate-status fallback). Any pending/missing/disagreement = not deployable. CANARY: gitea_pipeline now asserts the verdict evaluates needs results and fails closed — the class locked shut. 20 qualifier tests with fixtures from the real incident shapes; 1402 suite green.
153 lines
6.0 KiB
Python
153 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
|
"""deploy_qualify.py — read the CI qualification status for one commit.
|
|
|
|
Called by .gitea/workflows/deploy.yml (kept out of the YAML so the workflow
|
|
has no embedded multi-line scripts — a YAML block-scalar indentation hazard).
|
|
|
|
Prints one of: success | failure | pending
|
|
Exit code 0 always (the shell decides what to do with the verdict).
|
|
|
|
RELEASE-SAFETY HARDENING (2026-09-13, after red-CI promotions 0.66.28/29):
|
|
the verdict is the AND of two independent legs —
|
|
1. required-checks context (newest observation for this exact SHA);
|
|
2. the latest COMPLETED ci.yml@refs/heads/master run for this exact SHA,
|
|
every job concluded success (branch+tag fan-out resolved by path;
|
|
jobs envelope unwrapped — no aggregate-commit-status fallback, the
|
|
exact bug class verify.py hit in W/R-v0.3).
|
|
Any disagreement, pending, or missing evidence → pending/failure.
|
|
FAIL-CLOSED: not verified → not deployable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
import sys as _sys
|
|
_SCRIPT_DIR = str(__import__('pathlib').Path(__file__).resolve().parent)
|
|
if _SCRIPT_DIR not in _sys.path:
|
|
_sys.path.insert(0, _SCRIPT_DIR)
|
|
import gitea_auth
|
|
|
|
REQUIRED_CONTEXT = "CI Pipeline / required-checks"
|
|
CI_WORKFLOW_PATH = "ci.yml@refs/heads/master"
|
|
|
|
|
|
# ── pure verdict logic (unit-tested; fixtures from the real incidents) ────
|
|
|
|
def required_check_verdict(statuses: list[dict]) -> str:
|
|
"""Newest non-skipped required-checks observation for the SHA.
|
|
Missing → pending. Newest wins (stale success never overrides a newer
|
|
failure, and vice versa)."""
|
|
candidates = [
|
|
s for s in statuses
|
|
if str(s.get("context", "")).startswith(REQUIRED_CONTEXT)
|
|
and s.get("status") != "skipped"
|
|
]
|
|
if not candidates:
|
|
return "pending"
|
|
newest = max(candidates, key=lambda s: str(s.get("created_at", "")))
|
|
return str(newest.get("status", "pending"))
|
|
|
|
|
|
def independent_jobs_verdict(runs: list[dict], jobs_fetch, sha: str = "") -> str:
|
|
"""Latest COMPLETED ci.yml@refs/heads/master run for the exact SHA:
|
|
every job must conclude success. Fan-out-safe (tag runs and sibling
|
|
workflows are excluded by path); the run is resolved by head_sha, never
|
|
by 'latest branch state'. Missing run → pending. Any non-success
|
|
conclusion → failure."""
|
|
mine = [r for r in runs if r.get("path") == CI_WORKFLOW_PATH]
|
|
if sha:
|
|
mine = [r for r in mine if str(r.get("head_sha", "")) == sha]
|
|
completed = [r for r in mine if r.get("status") == "completed"]
|
|
if not completed:
|
|
return "pending"
|
|
latest = max(completed, key=lambda r: str(r.get("created_at", "") or r.get("run_started_at", "")))
|
|
jobs = jobs_fetch(latest.get("id"))
|
|
if jobs is None:
|
|
return "pending" # jobs probe failed → not verified
|
|
if not jobs:
|
|
return "pending" # no jobs visible → cannot verify
|
|
for j in jobs:
|
|
if j.get("conclusion") != "success":
|
|
return "failure"
|
|
return "success"
|
|
|
|
|
|
def combine(a: str, b: str) -> str:
|
|
"""Fail-closed conjunction: any failure → failure; any pending → pending;
|
|
success only when BOTH legs are success."""
|
|
if "failure" in (a, b):
|
|
return "failure"
|
|
if "pending" in (a, b):
|
|
return "pending"
|
|
return "success"
|
|
|
|
|
|
def qualify(statuses: list[dict], runs: list[dict], jobs_fetch, sha: str = "") -> str:
|
|
return combine(required_check_verdict(statuses),
|
|
independent_jobs_verdict(runs, jobs_fetch, sha=sha))
|
|
|
|
|
|
# ── API fetches (thin; injectable in tests) ───────────────────────────────
|
|
|
|
def _get_json(url: str, token: str, timeout: int = 30):
|
|
req = urllib.request.Request(url, headers={"Authorization": f"token {token}"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return json.loads(r.read())
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
|
|
def _fetch_status(server: str, repo: str, sha: str, token: str) -> list[dict] | None:
|
|
data = _get_json(f"{server}/api/v1/repos/{repo}/commits/{sha}/status", token)
|
|
return None if data is None else data.get("statuses", [])
|
|
|
|
|
|
def _fetch_runs(server: str, repo: str, sha: str, token: str) -> list[dict] | None:
|
|
data = _get_json(f"{server}/api/v1/repos/{repo}/actions/runs?limit=50", token)
|
|
if data is None:
|
|
return None
|
|
runs = data.get("workflow_runs", data if isinstance(data, list) else [])
|
|
return [r for r in runs if str(r.get("head_sha", "")) == sha]
|
|
|
|
|
|
def _fetch_jobs(server: str, repo: str, run_id, token: str) -> list[dict] | None:
|
|
data = _get_json(f"{server}/api/v1/repos/{repo}/actions/runs/{run_id}/jobs", token)
|
|
if data is None:
|
|
return None
|
|
# jobs envelope unwrap (this Gitea wraps in {"jobs": [...]}; some
|
|
# versions return a bare list) — never fall back to aggregate status
|
|
if isinstance(data, dict):
|
|
return data.get("jobs", [])
|
|
return data if isinstance(data, list) else None
|
|
|
|
|
|
# ── CLI ───────────────────────────────────────────────────────────────────
|
|
|
|
def main() -> int:
|
|
server = os.environ.get("GITEA_SERVER", "").rstrip("/")
|
|
repo = os.environ.get("GITEA_REPO", "")
|
|
sha = os.environ.get("DEPLOY_SHA", "")
|
|
token = gitea_auth.get_token() if gitea_auth.has_token() else ""
|
|
if not (server and repo and sha and token):
|
|
print("pending") # missing config → cannot verify → not qualified
|
|
return 0
|
|
|
|
statuses = _fetch_status(server, repo, sha, token)
|
|
runs = _fetch_runs(server, repo, sha, token)
|
|
if statuses is None or runs is None:
|
|
# transient API/network failure → keep polling
|
|
print("pending (probe failed)")
|
|
return 0
|
|
jobs_fetch = lambda run_id: _fetch_jobs(server, repo, run_id, token) # noqa: E731
|
|
verdict = qualify(statuses, runs, jobs_fetch, sha=sha)
|
|
print(verdict)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|