Nick
03070830b6
- warm_ablation.py: 4-condition matrix (lexical/hybrid/hybrid+warm/ hybrid+outcome) with the KEEP-vs-SIMPLIFY verdict rule - growth_delta.py: 8 behavioural metrics with advantage deltas (NOT database growth) - transfer_experiment.py: the both-models-must-clear protocol for the core research hypothesis (one-model wins don't count) - 9/9 green
66 lines
2.2 KiB
Python
66 lines
2.2 KiB
Python
#!/usr/bin/env python3
|
|
"""growth_delta.py — #46: Growth Delta as measured behavioural advantage.
|
|
|
|
The dashboard metrics (NOT database growth):
|
|
task_success, repeated_mistake_rate, owner_interventions,
|
|
correct_capability_selection, cloud_tokens_per_task, novel_transfer,
|
|
cross_model_transfer, personality_consistency.
|
|
|
|
Each metric: current value + baseline + delta (advantage, not vibes).
|
|
|
|
Usage: python3 scripts/growth_delta.py [--json]
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
|
|
# Offline fixture values (recorded; live measurement replaces).
|
|
METRICS = {
|
|
"task_success": {"current": 0.78, "baseline": 0.64},
|
|
"repeated_mistake_rate": {"current": 0.09, "baseline": 0.21},
|
|
"owner_interventions": {"current": 3, "baseline": 9},
|
|
"correct_capability_selection": {"current": 0.86, "baseline": 0.71},
|
|
"cloud_tokens_per_task": {"current": 1580, "baseline": 2400},
|
|
"novel_transfer": {"current": 0.62, "baseline": 0.41},
|
|
"cross_model_transfer": {"current": 0.58, "baseline": 0.40},
|
|
"personality_consistency": {"current": 0.91, "baseline": 0.83},
|
|
}
|
|
|
|
# lower-is-better metrics (delta sign flips)
|
|
LOWER_BETTER = {"repeated_mistake_rate", "owner_interventions",
|
|
"cloud_tokens_per_task"}
|
|
|
|
|
|
def delta():
|
|
rows = {}
|
|
for name, m in METRICS.items():
|
|
d = m["current"] - m["baseline"]
|
|
if name in LOWER_BETTER:
|
|
d = -d
|
|
rows[name] = {**m, "advantage": round(d, 3)}
|
|
overall = round(sum(r["advantage"] for r in rows.values()
|
|
if isinstance(r["advantage"], (int, float)))
|
|
/ len(rows), 3)
|
|
return {"metrics": rows, "growth_delta": overall,
|
|
"note": "behavioural advantage, NOT database growth"}
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--json", action="store_true")
|
|
args = ap.parse_args()
|
|
result = delta()
|
|
if args.json:
|
|
print(json.dumps(result, indent=1))
|
|
else:
|
|
for name, m in result["metrics"].items():
|
|
print(f"{name}: {m['current']} (baseline {m['baseline']}, "
|
|
f"advantage {m['advantage']:+})")
|
|
print("growth delta:", result["growth_delta"])
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
sys.exit(main())
|