rune/scripts/deploy_rust_build.py

#!/usr/bin/env python3
"""deploy_rust_build.py — #2455: the Rust cores ship as BINARIES, not sources.

Builds every Rust cdylib CI builds into the STAGED deployment tree (in-tree
target/release, exactly where the Python loaders look by default) and
smoke-checks one FFI operation per crate — BEFORE the symlink flips. The
deployed runtime then takes the Rust core by default instead of silently
running the Python fallbacks; the strangler's hardening/speed point finally
materializes in production.

Smoke tiers (honest, per crate):
  - symbol: CDLL-load the staged cdylib and resolve the crate's loader entry
    symbol (dlsym IS an FFI operation; proves the ELF links and exports).
  - call: a REAL FFI call through the staged loader itself (token_counting:
    tc_estimate_tokens({"text": ...}) -> int). One end-to-end call proves the
    full load path in the deployed revision.

Toolchain law (carried from ci.yml): rustup toolchain PATH-first (system
rustc is broken on this host — LLVM version-node symbol gap); builds run
in-tree so the cdylib lands at <crate>/target/release/lib<crate>.so.

Exit 0 = built + smoked; 1 = any failure (deploy_stage refuses promotion).
RUNE_DEPLOY_SKIP_RUST=1 is the documented emergency bypass — the deploy then
ships Python fallbacks as before and says so.
"""

from __future__ import annotations

import argparse
import ctypes
import json
import os
import subprocess
import sys
from pathlib import Path

RUSTUP_BIN = "/home/nick/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/bin"

# (crate, loader entry symbol, real-call payload or None)
CRATES: list[tuple[str, str, dict | None]] = [
    ("evidence_provenance", "ep_compute_hash", None),
    ("rune_spine", "sp_project", None),
    ("memory_scoring", "ms_compute_value", None),
    ("capability_scoring", "cs_score_card", None),
    ("token_counting", "tc_estimate_tokens", {"text": "ffi deploy smoke"}),
    ("scaffold_core", "sc_check_transition", None),
    ("memory_storage", "mems_read_query", None),
    ("memory_retrieval", "mr_call", None),
    ("cognition_core", "cc_default_budget", None),
    ("route_risk_flags", "rrf_detect", None),
    ("construction_interface", "ci_validate", None),
]


def _lib_path(staged_src: Path, crate: str) -> Path:
    return staged_src / "rust" / crate / "target" / "release" / f"lib{crate}.so"


def build_all(staged_src: Path) -> list[str]:
    built: list[str] = []
    env = dict(os.environ)
    env["PATH"] = f"{RUSTUP_BIN}:{env.get('PATH', '')}"
    env.pop("CARGO_TARGET_DIR", None)  # in-tree targets: the loaders' default path
    for crate, _sym, _payload in CRATES:
        manifest = staged_src / "rust" / crate / "Cargo.toml"
        if not manifest.exists():
            raise RuntimeError(f"{crate}: no Cargo.toml in the staged tree")
        r = subprocess.run(
            ["cargo", "build", "--release", "--manifest-path", str(manifest)],
            capture_output=True,
            text=True,
            env=env,
            timeout=600,
        )
        if r.returncode != 0:
            raise RuntimeError(f"{crate}: cargo build failed:\n{r.stderr[-600:]}")
        lib = _lib_path(staged_src, crate)
        if not lib.exists() or lib.stat().st_size < 1024:
            raise RuntimeError(f"{crate}: cdylib missing or implausibly small after build: {lib}")
        built.append(crate)
        print(f"  BUILT {crate}: {lib.stat().st_size} bytes")
    return built


def smoke_all(staged_src: Path) -> list[str]:
    smoked: list[str] = []
    for crate, sym, payload in CRATES:
        lib = ctypes.CDLL(str(_lib_path(staged_src, crate)))
        fn = getattr(lib, sym)  # dlsym: the FFI surface exists
        fn.restype = ctypes.c_void_p  # char* — never default c_int (loader law)
        if payload is not None:
            _real_call(staged_src, crate, payload)
            print(f"  SMOKED {crate}: symbol {sym} + real FFI call")
        else:
            print(f"  SMOKED {crate}: symbol {sym} resolved")
        smoked.append(crate)
    return smoked


def _real_call(staged_src: Path, crate: str, payload: dict) -> None:
    """Call the entry symbol directly with the loader's exact convention:
    argtypes c_char_p, restype c_void_p (char* — never default c_int), read
    the returned JSON envelope, free via <prefix>_free_string."""
    lib = ctypes.CDLL(str(_lib_path(staged_src, crate)))
    fn = getattr(lib, "tc_estimate_tokens")
    fn.argtypes = [ctypes.c_char_p]
    fn.restype = ctypes.c_void_p
    raw = json.dumps(payload).encode("utf-8")
    ptr = fn(raw)
    if not ptr:
        raise RuntimeError(f"{crate}: FFI call returned NULL")
    text = ctypes.cast(ptr, ctypes.c_char_p).value or b""
    getattr(lib, "tc_free_string")(ctypes.c_void_p(ptr))
    env = json.loads(text.decode("utf-8"))
    tokens = env.get("value", env.get("tokens")) if isinstance(env, dict) else env
    if not isinstance(tokens, int) or tokens <= 0:
        raise RuntimeError(f"{crate}: implausible FFI result: {text!r}")


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--staged", required=True, help="the staged tree (tmp_stage), pre-promotion")
    args = ap.parse_args()
    staged_src = Path(args.staged) / "src"

    print("RUST CORES INTO THE DEPLOYED REVISION (#2455)")
    built = build_all(staged_src)
    smoked = smoke_all(staged_src)
    missing = {c for c, *_ in CRATES} - set(smoked)
    if missing:
        raise RuntimeError(f"smoke coverage gap: {sorted(missing)}")
    print(f"OK: {len(built)} crates built into the staged revision, {len(smoked)} FFI-smoked")
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except SystemExit:
        raise
    except Exception as e:  # noqa: BLE001 — terminal reporter: prints the failure and exits 1 (never silent)
        print(f"deploy_rust_build: {e}", file=sys.stderr)
        raise SystemExit(1)