rune/scripts/deploy_promotion.py

#!/usr/bin/env python3
"""deploy_promotion.py — monotonic promotion authority for Rune deploys.

SCAFFOLD contract DEPLOY-MONOTONIC-001. Root cause of the 2026-09-13
deploy-order race (deploys 7355/7359): deploy_stage.py promoted
unconditionally in job-finish order — no comparison against the currently
deployed commit, no lock, no truthful outcome for a stale candidate.

This module owns the PROMOTION decision. The invariant:

    current never regresses along the canonical release branch.

Ordering authority: GIT ANCESTRY on the canonical release branch
(refs/heads/master). Rune's release flow is linear history — every deploy
candidate is a master push (tranche commit or its auto-bump descendant).
    candidate == deployed commit      -> NOOP_ALREADY_CURRENT (idempotent)
    candidate is DESCENDANT of it     -> allowed (normal forward deploy)
    candidate is ANCESTOR of it       -> STALE_SUPERSEDED (refuse; truthful)
    no ancestry relationship          -> REFUSED_DIVERGENT (fail closed)
    deployed commit unknown/unreadable-> REFUSED_UNKNOWN_CURRENT (fail closed)

Mechanism (smallest that provides the invariant, per brief §4-5):
    1. promotion-time stale check — ancestry comparison against the commit
       current ACTUALLY points to at promotion time (not at job start);
    2. an flock'd critical section around the read-compare-write of
       current/last-good so two concurrent promotions cannot interleave
       between the ancestry check and the symlink swap.
Workflow-level concurrency is NOT relied on (brief §5): the fence lives at
the promotion operation itself.

Truthful outcomes (brief §8) — build/stage success != promotion success:
    PROMOTED                candidate became current (verified below)
    NOOP_ALREADY_CURRENT    same commit already deployed; nothing mutated
    STALE_SUPERSEDED        a newer commit is already current; nothing mutated
    REFUSED_DIVERGENT       ordering unprovable; fail closed
    REFUSED_UNKNOWN_CURRENT deployed commit unreadable; fail closed
    FAILED_*                verification/staging failures (in deploy_stage)

A stale or refused outcome is a NON-ERROR terminal state for the release
pipeline (the newer build simply won) but MUST NOT print "PROMOTED" and
MUST NOT mutate current/last-good. deploy_stage's exit code and the
workflow's post-deploy runtime verification enforce this: the workflow
asserts current == candidate after a PROMOTED outcome, so a stale outcome
makes the job report honestly rather than masquerade.
"""

from __future__ import annotations

import fcntl
import json
import os
import subprocess
from pathlib import Path

DEPLOY_ROOT = Path("/mnt/workdrive/rune-code")
RELEASE_BRANCH = "refs/heads/master"

OUTCOME_PROMOTED = "PROMOTED"
OUTCOME_NOOP = "NOOP_ALREADY_CURRENT"
OUTCOME_STALE = "STALE_SUPERSEDED"
OUTCOME_DIVERGENT = "REFUSED_DIVERGENT"
OUTCOME_UNKNOWN_CURRENT = "REFUSED_UNKNOWN_CURRENT"


# ── pure decision logic (unit-tested; no I/O) ──────────────────────────────


def decide_promotion(candidate_sha: str, deployed_sha: str, ancestry: str) -> str:
    """Pure promotion decision from a precomputed ancestry relationship.
    ancestry is one of: 'same' | 'descendant' | 'ancestor' | 'divergent' |
    'unknown'. Fail-closed: anything not provably allowed is refused."""
    if candidate_sha == deployed_sha or ancestry == "same":
        return OUTCOME_NOOP
    if ancestry == "descendant":
        return OUTCOME_PROMOTED
    if ancestry == "ancestor":
        return OUTCOME_STALE
    if ancestry == "divergent":
        return OUTCOME_DIVERGENT
    return OUTCOME_UNKNOWN_CURRENT


def ancestry_of(candidate_sha: str, deployed_sha: str, repo: Path, fetch_on_divergent: bool = True) -> str:
    """Classify candidate vs deployed using git merge-base --is-ancestor on
    the canonical release branch's repository. Returns 'unknown' on any git
    failure (fail-closed callers treat that as refusal).

    Observed live 2026-09-13 (deploy 7449, SHA 535c2909): a DELAYED older
    candidate's checkout cannot contain commits newer than itself — its
    local object store lacks the deployed commit entirely, so ancestry
    classifies 'divergent' and fail-closed refuses. Correct refusal, wrong
    reason: the candidate IS an ancestor. When the local classification is
    divergent, fetch origin's canonical branch first (best-effort; the
    deploy workflow's checkout has credentials configured) and re-classify.
    If still divergent afterwards, the refusal stands with better
    evidence."""

    def _is_ancestor(a: str, b: str) -> bool:
        r = subprocess.run(
            ["git", "-C", str(repo), "-c", f"safe.directory={repo}", "merge-base", "--is-ancestor", a, b],
            capture_output=True,
            text=True,
        )
        return r.returncode == 0

    if candidate_sha == deployed_sha:
        return "same"
    if _is_ancestor(candidate_sha, deployed_sha):
        return "ancestor"  # candidate is OLDER than what's deployed
    if _is_ancestor(deployed_sha, candidate_sha):
        return "descendant"  # candidate is NEWER (normal forward deploy)
    if fetch_on_divergent:
        try:
            subprocess.run(
                ["git", "-C", str(repo), "-c", f"safe.directory={repo}", "fetch", "origin", "master", "--quiet"],
                capture_output=True,
                text=True,
                timeout=60,
            )
        except (OSError, subprocess.TimeoutExpired):
            pass  # best-effort: classification below still fails closed
        if _is_ancestor(candidate_sha, deployed_sha):
            return "ancestor"
        if _is_ancestor(deployed_sha, candidate_sha):
            return "descendant"
    return "divergent"


def read_deployed_commit(deploy_root: Path = DEPLOY_ROOT) -> str:
    """Read the commit the `current` symlink ACTUALLY points to right now —
    from the deployed build identity, the deployment authority (never from
    a stale job-start artifact). Empty string if unreadable."""
    current = deploy_root / "current"
    try:
        target = Path(os.readlink(current))
    except OSError:
        return ""
    identity = target / "src" / "rune" / "_build_identity.json"
    try:
        data = json.loads(identity.read_text(encoding="utf-8"))
        return str(data.get("commit", ""))
    except (OSError, ValueError):
        return ""


# ── the fenced promotion operation ──────────────────────────────────────────


def promote(
    candidate_sha: str,
    stage: Path,
    tmp_stage: Path,
    repo: Path,
    deploy_root: Path = DEPLOY_ROOT,
    on_promoted=None,
) -> tuple[str, str]:
    """Atomically decide + promote under an exclusive lock.

    Returns (outcome, detail). ONLY the PROMOTED outcome mutates
    current/last-good, and it does so inside the same critical section as
    the decision — no interleaving window between check and swap.

    Crash-safety ordering (brief §16): the stage rename happens BEFORE the
    lock (a staged-but-unpromoted tree is inert), the symlink swap is a
    single rename(2) via a temporary link, so current is never absent and
    never half-updated. last-good is refreshed BEFORE the swap only from a
    value that was genuinely current a moment ago.
    """
    deploy_root.mkdir(parents=True, exist_ok=True)
    lock_path = deploy_root / ".promotion.lock"
    with open(lock_path, "w") as lock_fd:
        fcntl.flock(lock_fd, fcntl.LOCK_EX)
        try:
            deployed = read_deployed_commit(deploy_root)
            if not deployed:
                return (OUTCOME_UNKNOWN_CURRENT, "deployed commit unreadable — fail closed")
            rel = ancestry_of(candidate_sha, deployed, repo)
            outcome = decide_promotion(candidate_sha, deployed, rel)
            if outcome != OUTCOME_PROMOTED:
                return (outcome, f"deployed={deployed[:12]} ancestry={rel}")

            # --- mutate (still inside the lock) ---
            current = deploy_root / "current"
            last_good = deploy_root / "last-good"
            prev_target = os.readlink(current) if current.is_symlink() else ""
            if prev_target and (deploy_root / "last-good").is_symlink():
                (deploy_root / "last-good").unlink()
            if prev_target:
                _atomic_symlink(deploy_root, "last-good", prev_target)
            if stage.exists():
                # re-deploying an already-staged SHA: restage over it
                import shutil

                shutil.rmtree(stage)
            os.rename(tmp_stage, stage)
            _atomic_symlink_target(deploy_root, "current", stage)
            detail = f"current -> {stage.name}"
            if on_promoted is not None:
                # §12C: race-safe convergence — the WINNER refreshes release
                # history atomically with its promotion, inside the lock.
                # Failure degrades to STALE (software valid, history lags;
                # retry by any later deploy converges idempotently).
                try:
                    on_promoted(candidate_sha)
                    detail += " (+ release-history converged)"
                except Exception as exc:  # noqa: BLE001
                    detail += f" (release-history refresh FAILED: {exc!s:.120} — STALE, retry converges)"
            return (OUTCOME_PROMOTED, detail)
        finally:
            fcntl.flock(lock_fd, fcntl.LOCK_UN)


def _atomic_symlink(deploy_root: Path, name: str, target: str) -> None:
    """Symlink swap that never leaves `name` absent: write temp link, rename
    over the final name (rename(2) is atomic on the same filesystem)."""
    _atomic_symlink_target(deploy_root, name, Path(target))


def _atomic_symlink_target(deploy_root: Path, name: str, target: Path) -> None:
    tmp = deploy_root / f".{name}.swap-{os.getpid()}"
    if tmp.is_symlink() or tmp.exists():
        tmp.unlink()
    tmp.symlink_to(target)
    os.replace(tmp, deploy_root / name)