317 lines
11 KiB
Python
317 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""growth_simulator.py — Simulate governed adaptation growth.
|
|
|
|
Runs synthetic queries against the memory system to:
|
|
1. Speed up proven_uses accumulation (synthetic washing)
|
|
2. Simulate weeks of usage in minutes
|
|
3. Find bugs and edge cases in the trust level lifecycle
|
|
4. Identify memories that never surface (dead weight)
|
|
5. Test gate logic under load
|
|
|
|
Usage:
|
|
growth_simulator.py simulate [--queries N] [--verbose]
|
|
growth_simulator.py analyze
|
|
growth_simulator.py stress [--queries N]
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import random
|
|
import sys
|
|
import time
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime, timezone
|
|
|
|
SCRIPTS_DIR = os.path.expanduser("~/.config/opencode/scripts")
|
|
sys.path.insert(0, SCRIPTS_DIR)
|
|
|
|
# Test queries spanning different topics and complexities
|
|
BROAD_QUERIES = [
|
|
"guitar", "Nick", "music", "coding", "AI", "memory", "persona",
|
|
"constitution", "voice", "identity", "project", "Delta Green",
|
|
"Nottingham", "REAPER", "Ollama", "MiMoCode", "TUI", "Rune",
|
|
"automation", "n8n", "website", "course", "YouTube",
|
|
]
|
|
|
|
SPECIFIC_QUERIES = [
|
|
"what exactly did Nick say about the guitar course timeline",
|
|
"how does the warm router scoring algorithm work",
|
|
"what is the drift-with-integrity framework",
|
|
"when was the last time Nick mentioned Delta Green",
|
|
"what are the 25 constitution rules",
|
|
"how does the curator decide what to promote",
|
|
"what model does Nick use for local inference",
|
|
"what is the four-layer memory architecture",
|
|
"how does the pre-commit gate work",
|
|
"what is the Daoist philosophical foundation",
|
|
]
|
|
|
|
MIXED_QUERIES = [
|
|
"Nick's guitar teaching schedule",
|
|
"Rune's voice influences",
|
|
"memory consolidation pipeline",
|
|
"Ollama model configuration",
|
|
"build mode complexity scorer",
|
|
"persona integrity check",
|
|
"claim audit verification",
|
|
"drift detection system",
|
|
"warm neuron scoring",
|
|
"knowledge graph entities",
|
|
]
|
|
|
|
|
|
def generate_queries_from_entries(db, limit=100):
|
|
"""Generate queries from actual entries in the store.
|
|
|
|
This ensures the simulation exercises entries that exist,
|
|
not just entries that match predefined queries.
|
|
"""
|
|
rows = db.execute(
|
|
"SELECT text FROM entries WHERE status='active' ORDER BY RANDOM() LIMIT ?",
|
|
(limit,)
|
|
).fetchall()
|
|
|
|
queries = []
|
|
for row in rows:
|
|
text = row[0] or ""
|
|
# Extract key words (3-5 words)
|
|
words = [w for w in text.split() if len(w) > 3][:5]
|
|
if words:
|
|
queries.append(" ".join(words))
|
|
|
|
return queries
|
|
|
|
|
|
def simulate_growth(db, num_queries=100, verbose=False):
|
|
"""Simulate growth by running queries and tracking which memories get accessed."""
|
|
import memory_store
|
|
import memory_consolidation
|
|
|
|
mc = memory_consolidation.MemoryConsolidation()
|
|
|
|
# Track results
|
|
results = {
|
|
"total_queries": 0,
|
|
"consolidated_hits": 0,
|
|
"full_retrieval_hits": 0,
|
|
"gate_decisions": {"consolidated": 0, "full": 0},
|
|
"entries_accessed": Counter(),
|
|
"entries_promoted": 0,
|
|
"trust_level_changes": defaultdict(int),
|
|
"errors": [],
|
|
}
|
|
|
|
all_queries = BROAD_QUERIES + SPECIFIC_QUERIES + MIXED_QUERIES
|
|
|
|
# Generate queries from actual entries to exercise the full store
|
|
generated = generate_queries_from_entries(db, limit=200)
|
|
all_queries = all_queries + generated
|
|
|
|
for i in range(num_queries):
|
|
query = random.choice(all_queries)
|
|
results["total_queries"] += 1
|
|
|
|
try:
|
|
# Test gate logic
|
|
gate = memory_store.gate_consolidated(db, query)
|
|
if gate.get("use_consolidated"):
|
|
results["gate_decisions"]["consolidated"] += 1
|
|
results["consolidated_hits"] += 1
|
|
else:
|
|
results["gate_decisions"]["full"] += 1
|
|
|
|
# Run recall
|
|
recall_results = memory_store.cached_recall(db, query)
|
|
if recall_results:
|
|
results["full_retrieval_hits"] += 1
|
|
for r in recall_results:
|
|
eid = r.get("id")
|
|
if eid:
|
|
results["entries_accessed"][eid] += 1
|
|
# Simulate proven access
|
|
memory_store.increment_wash(db, eid)
|
|
|
|
if verbose and i % 10 == 0:
|
|
print(f" Query {i+1}/{num_queries}: \"{query}\" → {len(recall_results)} results")
|
|
|
|
except Exception as e:
|
|
results["errors"].append(f"Query {i}: {str(e)[:100]}")
|
|
|
|
# Check trust level changes
|
|
rows = db.execute(
|
|
"SELECT rock_status, COUNT(*) FROM entries WHERE status='active' GROUP BY rock_status"
|
|
).fetchall()
|
|
for status, count in rows:
|
|
results["trust_level_changes"][status] = count
|
|
|
|
return results
|
|
|
|
|
|
def analyze_growth(db):
|
|
"""Analyze the current state of the governed adaptation system."""
|
|
import drift_core
|
|
|
|
# Trust level distribution
|
|
trust_dist = db.execute(
|
|
"SELECT rock_status, COUNT(*) FROM entries WHERE status='active' GROUP BY rock_status"
|
|
).fetchall()
|
|
|
|
# Proven uses distribution
|
|
proven_dist = db.execute(
|
|
"SELECT wash_count, COUNT(*) FROM entries WHERE status='active' AND wash_count > 0 GROUP BY wash_count ORDER BY wash_count DESC"
|
|
).fetchall()
|
|
|
|
# Dead weight: entries never accessed
|
|
total = db.execute("SELECT COUNT(*) FROM entries WHERE status='active'").fetchone()[0]
|
|
never_accessed = db.execute(
|
|
"SELECT COUNT(*) FROM entries WHERE status='active' AND (wash_count = 0 OR wash_count IS NULL)"
|
|
).fetchone()[0]
|
|
|
|
# Top proven entries
|
|
top_proven = db.execute(
|
|
"SELECT id, text, wash_count, rock_status FROM entries WHERE status='active' ORDER BY wash_count DESC LIMIT 10"
|
|
).fetchall()
|
|
|
|
# State evolution
|
|
vel = drift_core.compute_drift_velocity(db)
|
|
direction = drift_core.compute_drift_direction(db)
|
|
|
|
return {
|
|
"trust_distribution": {s: c for s, c in trust_dist},
|
|
"proven_distribution": {w: c for w, c in proven_dist[:10]},
|
|
"total_entries": total,
|
|
"never_accessed": never_accessed,
|
|
"dead_weight_pct": (never_accessed / total * 100) if total > 0 else 0,
|
|
"top_proven": [(r[0], r[1][:50], r[2], r[3]) for r in top_proven],
|
|
"velocity": vel,
|
|
"direction": direction,
|
|
}
|
|
|
|
|
|
def stress_test(db, num_queries=500):
|
|
"""Stress test: run many queries and find edge cases."""
|
|
import memory_store
|
|
|
|
errors = []
|
|
edge_cases = []
|
|
query_times = []
|
|
|
|
all_queries = BROAD_QUERIES + SPECIFIC_QUERIES + MIXED_QUERIES
|
|
|
|
for i in range(num_queries):
|
|
query = random.choice(all_queries)
|
|
start = time.time()
|
|
|
|
try:
|
|
results = memory_store.cached_recall(db, query)
|
|
elapsed = time.time() - start
|
|
query_times.append(elapsed)
|
|
|
|
# Edge case: empty results for broad query
|
|
if not results and query in BROAD_QUERIES:
|
|
edge_cases.append(f"Broad query \"{query}\" returned no results")
|
|
|
|
# Edge case: very slow query
|
|
if elapsed > 1.0:
|
|
edge_cases.append(f"Query \"{query}\" took {elapsed:.2f}s")
|
|
|
|
# Edge case: too many results
|
|
if len(results) > 20:
|
|
edge_cases.append(f"Query \"{query}\" returned {len(results)} results (potential noise)")
|
|
|
|
except Exception as e:
|
|
errors.append(f"Query {i}: {str(e)[:100]}")
|
|
|
|
return {
|
|
"total_queries": num_queries,
|
|
"errors": errors,
|
|
"edge_cases": edge_cases,
|
|
"avg_query_time": sum(query_times) / len(query_times) if query_times else 0,
|
|
"max_query_time": max(query_times) if query_times else 0,
|
|
"p95_query_time": sorted(query_times)[int(len(query_times) * 0.95)] if query_times else 0,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Growth simulator for governed adaptation")
|
|
sub = parser.add_subparsers(dest="cmd")
|
|
|
|
p_sim = sub.add_parser("simulate", help="Simulate growth with synthetic queries")
|
|
p_sim.add_argument("--queries", type=int, default=100, help="Number of queries")
|
|
p_sim.add_argument("--verbose", action="store_true")
|
|
|
|
sub.add_parser("analyze", help="Analyze current growth state")
|
|
|
|
p_stress = sub.add_parser("stress", help="Stress test the system")
|
|
p_stress.add_argument("--queries", type=int, default=500, help="Number of queries")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if not args.cmd:
|
|
parser.print_help()
|
|
sys.exit(1)
|
|
|
|
import memory_store
|
|
db = memory_store.connect()
|
|
|
|
if args.cmd == "simulate":
|
|
print(f"=== GROWTH SIMULATION ({args.queries} queries) ===\n")
|
|
results = simulate_growth(db, args.queries, args.verbose)
|
|
print(f"\nResults:")
|
|
print(f" Total queries: {results['total_queries']}")
|
|
print(f" Consolidated hits: {results['consolidated_hits']}")
|
|
print(f" Full retrieval hits: {results['full_retrieval_hits']}")
|
|
print(f" Gate decisions: consolidated={results['gate_decisions']['consolidated']}, full={results['gate_decisions']['full']}")
|
|
print(f" Entries accessed: {len(results['entries_accessed'])}")
|
|
print(f" Errors: {len(results['errors'])}")
|
|
print(f"\nTrust level distribution:")
|
|
for status, count in results['trust_level_changes'].items():
|
|
print(f" {status}: {count}")
|
|
if results['errors']:
|
|
print(f"\nErrors:")
|
|
for e in results['errors'][:5]:
|
|
print(f" {e}")
|
|
|
|
elif args.cmd == "analyze":
|
|
print("=== GROWTH ANALYSIS ===\n")
|
|
analysis = analyze_growth(db)
|
|
print(f"Trust level distribution:")
|
|
for status, count in analysis['trust_distribution'].items():
|
|
print(f" {status}: {count}")
|
|
print(f"\nProven uses distribution (top 10):")
|
|
for wash, count in analysis['proven_distribution'].items():
|
|
print(f" wash_count={wash}: {count} entries")
|
|
print(f"\nDead weight: {analysis['never_accessed']}/{analysis['total_entries']} ({analysis['dead_weight_pct']:.1f}%)")
|
|
print(f"\nTop proven entries:")
|
|
for eid, text, wash, status in analysis['top_proven']:
|
|
print(f" [{status}] {text} (wash={wash})")
|
|
print(f"\nState evolution:")
|
|
print(f" Velocity: {analysis['velocity']['changes_per_day']} changes/day")
|
|
print(f" Direction: {analysis['direction']['direction']}")
|
|
|
|
elif args.cmd == "stress":
|
|
print(f"=== STRESS TEST ({args.queries} queries) ===\n")
|
|
results = stress_test(db, args.queries)
|
|
print(f"Results:")
|
|
print(f" Total queries: {results['total_queries']}")
|
|
print(f" Errors: {len(results['errors'])}")
|
|
print(f" Edge cases: {len(results['edge_cases'])}")
|
|
print(f" Avg query time: {results['avg_query_time']:.4f}s")
|
|
print(f" Max query time: {results['max_query_time']:.4f}s")
|
|
print(f" P95 query time: {results['p95_query_time']:.4f}s")
|
|
if results['edge_cases']:
|
|
print(f"\nEdge cases:")
|
|
for ec in results['edge_cases'][:10]:
|
|
print(f" {ec}")
|
|
if results['errors']:
|
|
print(f"\nErrors:")
|
|
for e in results['errors'][:5]:
|
|
print(f" {e}")
|
|
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|