rune/scripts/gen_model_profiles_truth.py

#!/usr/bin/env python3
"""gen_model_profiles_truth.py — freeze model_profiles oracle truth via
the SHARED harness (scripts/freeze_truth.py, freeze()) through the
scripts/mp_freeze_seams rig (auto-imported sitecustomize: HOME pinned
to MP_FREEZE_HOME before the oracle import, per-case hermetic store
wipe+seed, canned Ollama server, JSON-safe wrapper seams).

Laws honoured:
  * hermetic: PROFILES_DIR is expanduser-derived AT IMPORT — the rig
    pins HOME to the fixed /tmp/mp_gap_freeze_home root (the CI_FREEZE_HOME
    precedent: frozen absolute paths stay byte-identical across processes).
  * no wall-clock: created/last_used come from the harness's frozen
    clock (write_envelope carries RUNE_FROZEN_NOW inside the file).
  * no network: scan_all_models runs against a canned in-process server
    (or a refused port for the degrade branch); OLLAMA_URL is repointed
    per case at call time.
  * self-contained rows: every stateful seam wipes + re-seeds the store
    from its args, so --verify replays byte-identically in any order.
  * byte-parity corpus: save_profile rows carry the RAW file text — the
    Go port's Python-shaped JSON writer must reproduce it exactly
    (asdict field order, indent=2 separators, float rendering, ASCII
    escaping). Seeds keep integral floats in float form (0.0) — the
    integral-float law; int-valued size_gb/optimal_temperature seeds
    render differently (Python keeps the int) and stay out of the corpus.

Regenerate + re-verify (byte-identical law — two runs, then verify):
  PYTHONPATH=scripts/mp_freeze_seams:src python3 scripts/gen_model_profiles_truth.py
  PYTHONPATH=scripts/mp_freeze_seams:src python3 scripts/freeze_truth.py \
      --verify src/go/model-profiles/frozen_mp_truth.json \
      --module rune.model_profiles
"""
import json
import os
import sys
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO / "scripts"))
sys.path.insert(0, str(REPO / "src"))

from freeze_truth import freeze, write_envelope  # noqa: E402  (shared harness)

FROZEN_NOW = "2026-10-13T09:41:23.123456"
OUT = REPO / "src" / "go" / "model-profiles" / "frozen_mp_truth.json"

FULL_STATE = {
    "name": "qwen2.5:7b",
    "display_name": "Qwen2.5 7B ✓",
    "family": "qwen",
    "parameters": "7B",
    "quantization": "Q4_K_M",
    "size_gb": 4.7,
    "strengths": ["reasoning", "coding", "instruction-following", "multilingual"],
    "weaknesses": ["verbose thinking"],
    "prompting_tips": [
        "Be direct", "Use structure", "Specify the language", "Ask for concise",
    ],
    "safety_tier": 2,
    "known_refusal_topics": ["weapons", "malware"],
    "thinking_format": "<think>",
    "response_style": "concise",
    "context_window": 32768,
    "optimal_temperature": 0.7,
    "interactions": 10,
    "successes": 7,
    "failures": 3,
    "last_used": "2026-10-12T00:00:00+00:00",
    "created": "2026-10-01T00:00:00+00:00",
}

QWEN_FILE = "qwen2.5_7b.json"

SCAN_FRESH = {"models": [
    {"name": "qwen2.5:7b", "size": 7516192768},
    {"name": "banker-tie-lo:2b", "size": 2415919104},   # 2.25 GB → 2.2 (ties-to-even)
    {"name": "banker-tie-hi:2b", "size": 2952790016},   # 2.75 GB → 2.8 (ties-to-even)
    {"size": 123},                                       # no name → skipped
    {"name": "noninteger:5b", "size": 5000000000},       # 4.656… → 4.7
]}

SCAN_EXISTING = {"models": [
    {"name": "qwen2.5:7b", "size": 7516192768},
    {"name": "llama3.1:8b-q4_K_M", "size": 4831838208},  # 4.5 GB
]}

CASES = [
    # ── get_profile_path: filesystem sanitization ────────────────────
    ("seam_get_profile_path", ("qwen2.5:7b",), {}),
    ("seam_get_profile_path", ("library/qwen3:14b",), {}),
    ("seam_get_profile_path", ("a?b:c/d",), {}),
    ("seam_get_profile_path", ("plain-model",), {}),

    # ── load_profile: disk → dataclass, suppressed-exception floor ───
    ("seam_load_profile", ("qwen2.5:7b", {QWEN_FILE: FULL_STATE}), {}),
    ("seam_load_profile", ("qwen2.5:7b", None), {}),                      # no file → None
    ("seam_load_profile", ("qwen2.5:7b", {"other_7b.json": FULL_STATE}), {}),  # wrong file → None
    ("seam_load_profile", ("qwen2.5:7b", {QWEN_FILE: "{not json"}), {}),   # corrupt → None
    ("seam_load_profile", ("qwen2.5:7b", {QWEN_FILE: {                      # unknown keys filtered
        "name": "qwen2.5:7b", "bogus": 1, "family": "qwen"}}), {}),
    ("seam_load_profile", ("qwen2.5:7b", {QWEN_FILE: {                      # missing optionals → defaults
        "name": "qwen2.5:7b", "interactions": 5}}), {}),
    ("seam_load_profile", ("qwen2.5:7b", {QWEN_FILE: {"family": "qwen"}}), {}),        # missing name → TypeError suppressed

    # ── save_profile: BYTE-PARITY corpus (raw file text) ─────────────
    ("seam_save_profile", (FULL_STATE,), {}),
    ("seam_save_profile", ({"name": "plain-model"},), {}),                 # all defaults
    ("seam_save_profile", ({
        "name": "deepseek-r1:32b", "family": "deepseek", "parameters": "32B",
        "size_gb": 0.0, "strengths": [], "safety_tier": 3,
        "optimal_temperature": 0.6,
    },), {}),

    # ── build_profile_from_ollama: knowledge merge + clock + tier ────
    ("seam_build_profile_from_ollama", ("qwen3:14b", {"display_name": "Qwen3 14B", "size_gb": 9.0}), {}),
    ("seam_build_profile_from_ollama", ("mistral-7b", None), {}),
    ("seam_build_profile_from_ollama", ("dolphin-uncensored:7b", None), {}),
    ("seam_build_profile_from_ollama", ("DAVIDAU-test:14b", None), {}),
    ("seam_build_profile_from_ollama", ("some-new-model", {"display_name": "X"}), {}),
    ("seam_build_profile_from_ollama", ("deepseek-r1:32b", {"display_name": "DS", "size_gb": 19.2}), {}),

    # ── scan_all_models: Ollama sweep (canned server / dead port) ────
    ("seam_scan_all_models", (SCAN_FRESH,), {}),
    ("seam_scan_all_models", (SCAN_EXISTING, {QWEN_FILE: FULL_STATE}), {}),  # existing → learned data kept
    ("seam_scan_all_models", ({"models": [{"name": "phi3:mini"}]}, None), {}),  # no size → 0.0
    ("seam_scan_all_models", ("dead", None), {}),

    # ── get_or_create_profile: load-or-build-and-save ────────────────
    ("seam_get_or_create_profile", ("qwen2.5:7b", {QWEN_FILE: FULL_STATE}), {}),
    ("seam_get_or_create_profile", ("brand-new-model", None), {}),

    # ── record_interaction: learning accumulator (state = the contract)
    ("seam_record_interaction", ("qwen3:14b", [[True, ""]], None), {}),
    ("seam_record_interaction", ("fresh-fail:1b", [[False, "refuses weapons"]], None), {}),
    ("seam_record_interaction", ("dup-topic:1b", [[False, "t1"], [False, "t1"]], None), {}),
    ("seam_record_interaction", ("no-topic:1b", [[False, ""]], None), {}),
    ("seam_record_interaction", ("qwen2.5:7b", [[False, "politics"]], {QWEN_FILE: FULL_STATE}), {}),

    # ── get_prompt_context: operator string assembly ─────────────────
    ("seam_get_prompt_context", ("qwen2.5:7b", {QWEN_FILE: FULL_STATE}), {}),
    ("seam_get_prompt_context", ("tips-only:1b", {"tips-only_1b.json": {
        "name": "tips-only:1b", "prompting_tips": ["tip one"]}}), {}),
    ("seam_get_prompt_context", ("unknown-model-x", None), {}),             # fresh unknown → ""

    # ── to_dict / from_dict (dataclass surface) ──────────────────────
    ("seam_to_dict", (FULL_STATE,), {}),
    ("seam_from_dict", ({"name": "n", "bogus_key": 1, "context_window": 8192},), {}),
]


def main() -> int:
    # The harness clock pin is env-driven (RUNE_FROZEN_NOW) — set it for
    # the generation run too, not just --verify (write_envelope then
    # carries the same instant for byte-identical re-freezes).
    os.environ["RUNE_FROZEN_NOW"] = FROZEN_NOW
    truth = freeze("model_profiles", CASES, oracle_module="rune.model_profiles")
    write_envelope(FROZEN_NOW, truth, OUT)
    print(f"froze {len(truth)} cases -> {OUT}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())