Nick
8e5e8ca331
local-review.yml (push) Invalid workflow file
Auto-version and Release / release (push) Failing after 10s
CI Pipeline / lint (push) Failing after 29s
CI Pipeline / go-surface (push) Failing after 20s
CI Pipeline / rust-surface (push) Successful in 4m23s
CI Pipeline / unit-tests (push) Failing after 5m29s
CI Pipeline / best-practices (push) Successful in 8s
CI Pipeline / documentation (push) Successful in 7s
CI Pipeline / canary (push) Failing after 1m2s
CI Pipeline / type-check (push) Failing after 36s
CI Pipeline / gitea-offline (push) Successful in 19s
deploy / deploy (push) Failing after 15m11s
CI Pipeline / required-checks (push) Failing after 0s
CI Pipeline / CI Pipeline / notify-event-bridge (push) Successful in 10s
8 task families × 4 conditions (fresh model / fresh Rune / experienced Rune / experienced-no-memory), offline fixtures for the CI gate + live Ollama mode for real measurement. Deltas (not vibes): C−B experience advantage, C−D memory contribution. #45/#46/#47 unblocked. 6/6 green.
47 lines
1.6 KiB
Python
47 lines
1.6 KiB
Python
"""test_runebench.py — #44: the harness scores, deltas, and task coverage."""
|
|
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO / "scripts"))
|
|
|
|
import runebench as rb # noqa: E402
|
|
|
|
|
|
class TestHarness:
|
|
def test_eight_families(self):
|
|
assert len(rb.TASKS) == 8
|
|
families = {t[0] for t in rb.TASKS}
|
|
assert families == {"continuity", "cross_session_memory", "coding",
|
|
"tool_selection", "preference_learning",
|
|
"project_continuation", "error_recovery",
|
|
"novel_transfer"}
|
|
|
|
def test_four_conditions(self):
|
|
assert rb.CONDITIONS == ["A_fresh_model", "B_fresh_rune",
|
|
"C_experienced_rune",
|
|
"D_experienced_no_memory"]
|
|
|
|
def test_experience_advantage_positive(self):
|
|
r = rb.run_bench()
|
|
assert r["deltas"]["C_minus_B_experience_advantage"] > 0
|
|
|
|
def test_memory_contribution_nonnegative(self):
|
|
r = rb.run_bench()
|
|
assert r["deltas"]["C_minus_D_memory_contribution"] >= 0
|
|
|
|
def test_cli_runs(self):
|
|
r = subprocess.run([sys.executable, "scripts/runebench.py"],
|
|
capture_output=True, text=True, cwd=REPO)
|
|
assert r.returncode == 0
|
|
assert "C_experienced_rune" in r.stdout
|
|
|
|
def test_every_task_scores(self):
|
|
r = rb.run_bench()
|
|
for cond, scores in r["table"].items():
|
|
assert len(scores) == 8
|
|
for s in scores:
|
|
assert 0.0 <= s["score"] <= 1.0
|