rune/scripts/dichotomy_audit.py

#!/usr/bin/env python3
"""dichotomy_audit.py — the honest completion instrument for the dichotomy rebase.

Function-level, not directory-level: a module counts as covered only when its
PYTHON FUNCTIONS have identifiable counterparts in the Go/Rust implementation.
A Go directory with 3 structs and none of the module's 70 functions is a
facade, and this audit names it as one.

Matching rule (name-preserving ports): a Python function name matches when
its snake_case form, or its Go/Rust camelCase/PascalCase form, appears in the
candidate implementation sources. Deliberately strict — novel Go names that
don't echo the Python names do NOT count, because unverifiable "trust me,
it's in there somewhere" is exactly the failure mode this instrument exists
to kill.

Usage: python3 scripts/dichotomy_audit.py [--write]
  --write  regenerates docs/dichotomy-rebase-honest-audit.md
"""
from __future__ import annotations

import ast
import json
import re
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
SRC = ROOT / "src" / "rune"
GO = ROOT / "src" / "go"
RUST = ROOT / "src" / "rust"
REPORT = ROOT / "docs" / "dichotomy-rebase-honest-audit.md"
CLASSIFICATION = ROOT / "store" / "rebase" / "classification.json"

TEST_PREFIXES = ("test_",)
# Modules that stay Python by decision (test infra stays pytest)
TEST_INFRA = {
    "conftest", "test_truth", "test_adaptive_router", "test_canaries",
}
RETIRES = {"memory_env", "__main__"}
# The *_loader family is the programme's OWN ctypes FFI machinery — the
# delegation glue created by the ports themselves (broker_loader.py et al).
# Per the strangler-fig deletion doctrine they burn with the Python facades
# at stage 3; porting a loader to Rust is circular. Machinery, not port
# surface — same category as dunders.
FFI_MACHINERY = ("_loader",)


def snake_to_camel(name: str) -> list[str]:
    """Candidate identifier forms for a Python name in Go/Rust sources."""
    parts = name.split("_")
    camel = parts[0] + "".join(p.capitalize() for p in parts[1:])
    pascal = "".join(p.capitalize() for p in parts)
    return [name, camel, pascal]


def module_functions(path: Path) -> list[str]:
    """Every function and method name defined in the module.

    Machinery exclusions (same category as dunders — not port surface):
    - loader probes: _core, _go_kernel, _rust_core, _kernel (delegation glue;
      their whole job is resolving the kernel — porting them is circular)
    - module-prefixed loader probes: _sr_core (system_registry — delegates
      to sr_loader.core(), where the RUNE_SYSTEM_REGISTRY_CORE pin is
      honored), _rust (toolchain — dispatches getattr(tl_loader, fn) with
      the RUNE_TC_CORE pin and the _MISSING fallback); same category as
      _core, proved by the literal-100% L1 lane)
    - Python fallback pins: _py_* (strangler-fig stage-3 burn candidates —
      the kernel counterpart exists by design; counting them as 'unported'
      double-counts the work and reads PARTIAL forever)
    """
    try:
        tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"))
    except SyntaxError:
        return []
    MACHINERY = {"_core", "_go_kernel", "_rust_core", "_kernel", "_sr_core", "_rust",
                 # module-prefixed loader probes (self_improve agent finding
                 # 2026-10-14 + L1 lane proof: '_si_core'-class names are the
                 # same machinery as '_core' — env-pin check + kernel
                 # resolution; oracle body is literally the pin check)
                 "_si_core", "_smod_core", "_ta_core", "_mse_core", "_mr_core",
                 "_ss_core", "_stm_core", "_sg_core", "_bh_core", "_pe_core",
                 "_vl_core", "_ct_core", "_ca_core", "_cp_core", "_cc_core",
                 "_cs_core", "_sc_core", "_sck_core", "_gp_core", "_wr_core"}
    names = []
    for node in ast.walk(tree):
        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
            if node.name.startswith("__") and node.name.endswith("__"):
                continue  # dunders are runtime machinery, not port surface
            if node.name in MACHINERY or node.name.startswith("_py_"):
                continue  # loader probes + Python fallback pins
            names.append(node.name)
    return names


def load_classification() -> dict:
    if CLASSIFICATION.exists():
        return json.loads(CLASSIFICATION.read_text())
    return {}


def implementation_corpus() -> dict[str, str]:
    """module-kebab -> concatenated Go/Rust source text for that module."""
    corpus: dict[str, str] = {}

    for d in sorted(GO.iterdir()):
        if not d.is_dir():
            continue
        chunks = []
        for f in d.rglob("*.go"):
            if f.name.endswith("_test.go"):
                continue
            chunks.append(f.read_text(encoding="utf-8", errors="replace"))
        if chunks:
            corpus[d.name] = "\n".join(chunks)

    # Survivor packages (per-crate Go membranes under cognition/)
    cog = GO / "cognition"
    if cog.exists():
        for d in sorted(cog.iterdir()):
            if d.is_dir():
                chunks = [f.read_text(encoding="utf-8", errors="replace")
                          for f in d.rglob("*.go") if not f.name.endswith("_test.go")]
                if chunks:
                    corpus[f"cognition-{d.name}"] = "\n".join(chunks)

    for d in sorted(RUST.iterdir()):
        src = d / "src"
        if src.exists():
            # A module may have BOTH a Go and a Rust implementation
            # (e.g. verification_layer: full Go port + partial Rust
            # crate). Concatenate — never let one side shadow the other.

            chunks = [f.read_text(encoding="utf-8", errors="replace")
                      for f in src.rglob("*.rs")]
            if chunks:
                corpus[d.name] = corpus.get(d.name, "") + "\n" + "\n".join(chunks)
    return corpus


def module_to_keys(module_stem: str) -> list[str]:
    """Candidate implementation keys for a Python module stem."""
    kebab = module_stem.replace("_", "-")
    keys = [kebab, module_stem]
    # Hyphenated legacy twins (drift-ledger.py vs drift_ledger.py) share ONE
    # implementation; the underscore form is the crate/dir name.
    if "-" in module_stem:
        keys.append(module_stem.replace("-", "_"))
    # Deep ports and multi-module binaries register aliases:
    aliases = {
        "rune_operator": ["operator", "operator-deep", "operator-support", "core-server"],
        "rune_tui": ["tui"],
        "rune_operator_cli": ["operator"],
        # curator was ported to FULL Go (src/go/curator-core, a95c6ce1);
        # without this alias the audit mislabels it NOT TOUCHED.
        "curator": ["curator-core"],
        "memory_store": ["memory-ops", "memory-store-deep"],
        "warm_router": ["warm-router", "operator-support"],
        "tool_executor": ["tool-executor", "local-tool-runtime", "tool-protocol", "tool-protocol-rw"],
        "local_tool_runtime": ["local-tool-runtime", "operator"],
        "provider_registry": ["provider-registry", "model-connect"],
        "adaptive_router": ["adaptive-router", "operator-support"],
        "hook_engine": ["hook-engine"],
        "event_bus": ["event-bus"],
        "event_envelope": ["event-bus"],
        "message_queue": ["message-queue"],
        "subsystem_health": ["subsystem-health"],
        "parallel_processor": ["parallel-processor"],
        "connection_pool": ["connection-pool"],
        "recall_router": ["recall-router"],
        "cron_engine": ["cron-engine"],
        "cache": ["cache"],
        "night_mode": ["night-mode"],
        "work_manager": ["work-manager"],
        "failure_auto_filer": ["failure-filer"],
        "microkernel": ["microkernel"],
        "construction_interface": ["cognition-cognition", "cognition"],
        "construction_decision": ["cognition-cognition", "cognition"],
        "rune_spine": ["cognition-spine", "cognition"],
        "spine": ["cognition-spine", "cognition", "rune_spine"],
        "memory_storage": ["cognition-storage", "cognition"],
        "memory_retrieval": ["cognition-retrieval", "cognition"],
        "scaffold_core": ["cognition-scaffold", "cognition"],
        "evidence_provenance": ["cognition-provenance", "cognition"],
        "memory_scoring": ["cognition-scoring", "cognition"],
        "capability_scoring": ["cognition-capability", "cognition"],
        "curator": ["curator-core"],
        "token_counting": ["cognition-token", "cognition"],
        "query_memory": ["cognition-query", "cognition"],
        # drift_compression's kernels live in the memory_scoring crate
        # (H3 slice 2 + the stats kernel): the live delegation chain is
        # drift_compression → drift_core._ms_core → ms_loader → .so.
        "drift_compression": ["memory_scoring"],
        # audit_ledger's chain kernels live in the evidence_provenance
        # crate (Phase G slice 2, live ep seam — the oracle already
        # delegates hash+verify there): compute_hash/append_entry/
        # verify_chain/get_entries + the log_* gap-close wrappers + the
        # main() CLI mirror. The W6-era src/rust/audit_ledger stub
        # (SHA-256 concat, wrong field names — never the live chain) was
        # deleted; one chain, one implementation (RECONCILE-DON'T-CLOBBER).
        "audit_ledger": ["evidence_provenance"],
    }
    if module_stem in aliases:
        keys = aliases[module_stem] + keys
    # cognition_core broker.rs hosts many kernels
    if module_stem in {
        "tier_manager", "safety_override", "pins", "identity_anchors",
        "abstraction", "growth_delta", "witness", "persona_gate",
        "memory_consolidation", "context_broker",
        "persona_definition", "goal",
    }:
        keys.append("cognition_core")
        keys.append("cognition-cognition")
    return keys


def audit() -> dict:
    classification = load_classification()
    corpus = implementation_corpus()

    rows = []
    for path in sorted(SRC.glob("*.py")):
        stem = path.stem
        if stem in RETIRES:
            rows.append({"module": stem, "lines": len(path.read_text(errors='replace').splitlines()),
                         "fns": 0, "matched": 0, "side": "RETIRES", "status": "RETIRES", "missing": []})
            continue
        if stem.endswith(FFI_MACHINERY) or stem == "ffi_query_memory_smoke":
            rows.append({"module": stem, "lines": len(path.read_text(errors='replace').splitlines()),
                         "fns": 0, "matched": 0, "side": "FFI-MACHINERY", "status": "RETIRES", "missing": []})
            continue
        if stem.startswith(TEST_PREFIXES) or stem in TEST_INFRA:
            rows.append({"module": stem, "lines": len(path.read_text(errors='replace').splitlines()),
                         "fns": 0, "matched": 0, "side": "TEST", "status": "TEST", "missing": []})
            continue
        fns = module_functions(path)
        lines = len(path.read_text(errors="replace").splitlines())
        entry = classification.get(stem) or classification.get(path.name) or {}
        side = entry.get("side", entry.get("language", "unclassified"))

        sources = []
        for key in module_to_keys(stem):
            if key in corpus:
                sources.append(corpus[key])
        blob = "\n".join(sources)

        matched, missing = 0, []
        for fn in fns:
            hits = any(cand in blob for cand in snake_to_camel(fn))
            if hits:
                matched += 1
            else:
                missing.append(fn)

        ratio = matched / len(fns) if fns else (1.0 if lines < 30 else 0.0)
        if stem in RETIRES:
            status = "RETIRES"
        elif not blob:
            status = "NOT TOUCHED"
        elif ratio >= 0.8:
            status = "FULL"
        elif ratio >= 0.2:
            status = "PARTIAL"
        else:
            status = "NOT TOUCHED"
        rows.append({"module": stem, "lines": lines, "fns": len(fns),
                     "matched": matched, "side": side, "status": status, "missing": missing[:12]})

    return {"rows": rows}


def summarize(result: dict) -> str:
    rows = result["rows"]
    runtime = [r for r in rows if r["status"] not in ("TEST", "RETIRES")]
    full = [r for r in runtime if r["status"] == "FULL"]
    partial = [r for r in runtime if r["status"] == "PARTIAL"]
    untouched = [r for r in runtime if r["status"] == "NOT TOUCHED"]
    total_fns = sum(r["fns"] for r in runtime)
    total_matched = sum(r["matched"] for r in runtime)
    total_lines = sum(r["lines"] for r in runtime)
    fn_pct = 100 * total_matched / total_fns if total_fns else 0

    out = []
    out.append("# DICHOTOMY REBASE — HONEST AUDIT (function-level)")
    out.append("")
    out.append("_Regenerate with `python3 scripts/dichotomy_audit.py --write`. A module is FULL only when 80%+ of its Python functions have name-preserving counterparts in the Go/Rust implementation — directories without the functions are facades and count as NOT TOUCHED._")
    out.append("")
    # HEADLINE (Nick, 2026-10-13): progress is measured in FINISHED modules
    # (FULL), not half-done function matches — a percentage that mixes
    # complete and in-flight work flatters progress.
    total_mods = len(full) + len(partial) + len(untouched)
    mod_pct = 100 * len(full) / total_mods if total_mods else 0
    out.append(f"**FULL completion: {mod_pct:.0f}%** ({len(full)}/{total_mods} runtime modules at FULL)")
    out.append("")
    out.append(f"Function-level detail: {fn_pct:.0f}% ({total_matched}/{total_fns} runtime functions matched) — secondary; includes in-flight work")
    out.append("")
    out.append("| Status | Modules | Lines | Meaning |")
    out.append("|---|---|---|---|")
    out.append(f"| FULL | {len(full)} | {sum(r['lines'] for r in full)} | 80%+ of functions ported |")
    out.append(f"| PARTIAL | {len(partial)} | {sum(r['lines'] for r in partial)} | 20–80% — real logic exists, gaps named below |")
    out.append(f"| NOT TOUCHED | {len(untouched)} | {sum(r['lines'] for r in untouched)} | no name-preserving implementation |")
    out.append(f"| Total runtime | {len(runtime)} | {total_lines} | |")
    out.append("")

    if untouched:
        out.append("## NOT TOUCHED — the port queue, in priority order (largest first)")
        out.append("")
        out.append("| Module | Lines | Fns | Side |")
        out.append("|---|---|---|---|")
        for r in sorted(untouched, key=lambda r: -r["lines"]):
            out.append(f"| {r['module']} | {r['lines']} | {r['fns']} | {r['side']} |")
        out.append("")
    if partial:
        out.append("## PARTIAL — gaps to close (matched/total fns; first missing named)")
        out.append("")
        out.append("| Module | Lines | Coverage | Missing (first named) |")
        out.append("|---|---|---|---|")
        for r in sorted(partial, key=lambda r: -(r["fns"] - r["matched"])):
            miss = ", ".join(r["missing"][:6]) if r["missing"] else "(structural)"
            out.append(f"| {r['module']} | {r['lines']} | {r['matched']}/{r['fns']} | {miss} |")
        out.append("")
    out.append("## FULL — ported, parity-gated")
    out.append("")
    out.append("| Module | Lines | Fns | Side |")
    out.append("|---|---|---|---|")
    for r in sorted(full, key=lambda r: -r["lines"]):
        out.append(f"| {r['module']} | {r['lines']} | {r['matched']}/{r['fns']} | {r['side']} |")
    out.append("")
    return "\n".join(out)


def main() -> int:
    write = "--write" in sys.argv
    result = audit()
    report = summarize(result)
    if write:
        REPORT.write_text(report)
        print(f"written: {REPORT}")
    # stderr summary for the session log
    rows = result["rows"]
    runtime = [r for r in rows if r["status"] not in ("TEST", "RETIRES")]
    tf = sum(r["fns"] for r in runtime)
    tm = sum(r["matched"] for r in runtime)
    n_full = sum(1 for r in runtime if r['status'] == 'FULL')
    n_total = len(runtime)
    print(f"FULL: {100*n_full/n_total if n_total else 0:.0f}% ({n_full}/{n_total} modules) | "
          f"fn-level: {tm}/{tf} ({100*tm/tf if tf else 0:.0f}%) | "
          f"PARTIAL {sum(1 for r in runtime if r['status']=='PARTIAL')} | "
          f"NOT TOUCHED {sum(1 for r in runtime if r['status']=='NOT TOUCHED')}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())