fix(curator): recover the two provenance phantoms — lexicon_evolution + drift-ledger (contract BUILDER-2026-10-07T16:26:19Z-71624c)

Provenance over reconstruction (cycle-5 law):
- lexicon_evolution.py: ~/odysseus 0308ccc6 (feat/memory-lifecycle), verbatim
  229 lines — the branch HEAD had DELETED it, which is why the import was a
  phantom everywhere; no reconstruction was needed
- drift-ledger.py: ~/odysseus rework/product-api HEAD 8524e8e1 — the later
  evolution of the original (adds the package-relative import fallback that
  makes the curator's importlib load robust)

Restored semantics:
- le_promote_eligible now returns real lexicon verdicts (distinctive-term
  sharing / novel-axis) instead of the strength-only degrade
- _contradicts_identity runs real coherence conflict verdicts
- the drift write-gate (run_plan) loads and enforces (check strict=False rc=0)

Verified live: both modules import; curator call sites return lexicon
verdicts (degrade string absent); drift gate check passes; 93 related
tests green; ruff clean.
This commit is contained in:
Nick committed 2026-10-07 17:28:08 +01:00
1 parent 3199e36cd4
commit 574902926e
3 files changed
+6073 -2

No files matched your search

+355
View File
@@ -0,0 +1,355 @@
#!/usr/bin/env python3
"""Drift ledger — drift protection for long projects.
A persistent checksum ledger of the five core blocks. Each sleep-time / compaction
run records the sha256 and byte length of every block. Two signals are tracked:
- CUMULATIVE VOLUME: how far each block has moved from its *anchor* (a ledger
record labelled with a source directive or an explicit user confirmation).
Blocks near/over the threshold without an anchor source are flagged.
- SOURCE TYPE: how each byte of change is justified. Legitimate growth carries
a journaled operation (a user directive, a 3-sighting ledger promotion,
an ADD/UPDATE/DELETE applied by the curator). Unexplained bulk is drift.
The ANGLE of this file: growth belongs to the *harness* (blocks, ledger, skills,
rules), NOT the engine (the swappable LLM). When the model swaps, the harness and
its drift bounds must persist untouched. This ledger is part of the harness.
Usage:
drift-ledger.py snapshot # record block states, return JSON
drift-ledger.py check [--strict] # compare to last snapshot; emit DRIFT-ALERT
drift-ledger.py anchor <label> <reason> # re-anchor a block after a
# confirmed, authorised change
Exit 0 = healthy, 1 = drift flagged.
"""
import argparse
import hashlib
import json
import os
import sys
from datetime import datetime, timezone
import sys, os
_SD = os.path.dirname(os.path.abspath(__file__))
if _SD not in sys.path: sys.path.insert(0, _SD)
try:
from . import memory_env
except ImportError:
import memory_env
INDEX_FILE = os.path.join(memory_env.memory_dir(), "index", "blocks_meta.json")
DB = memory_env.store_db()
def ledger_path():
# The ledger lives beside the index, so overriding INDEX_FILE (as the
# curator gate and the canary do for isolation) moves the ledger too.
return os.path.join(os.path.dirname(INDEX_FILE), "drift_ledger.json")
JOURNAL_DIR = os.path.join(memory_env.memory_dir(), "journal")
# Max cumulative byte drift from an anchored/authorised state before we refuse.
DRIFT_LIMIT = 0.30 # 30% of block size
# Blocks whose changes are always allowed to grow fast (their job is state).
FAST_BLOCKS = ("project", "operating")
# The constitution is even more constrained — it changes ONLY by directive.
CONSTITUTION_LIMIT = 0.05 # 5% — anything more than a directive entry is drift
BLOCKS = ("constitution", "persona", "human", "operating", "project")
def _sha(text):
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def read_block(label):
"""Read a core block from the STORE (new schema) — the always-on entries
for that topic, serialized as text. The old markdown files are retired;
the store is the source of truth, so drift protection guards store content.
"""
try:
import sqlite3
db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
rows = db.execute(
"SELECT text FROM entries WHERE topic=? AND always_on=1 "
"AND status='active' ORDER BY priority ASC, id ASC", (label,)
).fetchall()
db.close()
return "\n".join(r[0] for r in rows)
except Exception:
return ""
def load_ledger():
if os.path.exists(ledger_path()):
with open(ledger_path()) as f:
return json.load(f)
return {}
def save_ledger(ledger):
path = ledger_path()
os.makedirs(os.path.dirname(path), exist_ok=True)
tmp = path + ".tmp"
with open(tmp, "w") as f:
json.dump(ledger, f, indent=2)
os.replace(tmp, path)
def journal(msg):
os.makedirs(JOURNAL_DIR, exist_ok=True)
month = datetime.now(timezone.utc).strftime("%Y-%m")
ts = datetime.now(timezone.utc).isoformat(timespec="seconds")
with open(os.path.join(JOURNAL_DIR, f"{month}.md"), "a") as f:
f.write(f"`{ts}` **DRIFT-ALERT** → harness\n - {msg}\n\n")
def snapshot():
ledger = load_ledger()
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
entry = {"when": now, "blocks": {}}
for label in BLOCKS:
raw = read_block(label)
entry["blocks"][label] = {
"sha": _sha(raw),
"size": len(raw),
"content": raw, # last-good content backup for diffing + retro-fix
}
ledger["last_snapshot"] = entry
save_ledger(ledger)
return entry
def anchor(label, reason):
"""Record that the current state of `label` is authorised (a user directive,
a confirmed review, a 3-sighting promotion). Subsequent drift is measured
from here — this is the re-anchor after legitimate change."""
ledger = load_ledger()
raw = read_block(label)
ledger.setdefault("anchors", {})[label] = {
"sha": _sha(raw),
"size": len(raw),
"content": raw, # authorised content backup for root-cause diffing
"reason": reason,
"when": datetime.now(timezone.utc).isoformat(timespec="seconds"),
}
save_ledger(ledger)
return f"anchored {label}: {reason}"
def check(strict=False, autofix=True):
ledger = load_ledger()
snap = ledger.get("last_snapshot")
if not snap:
# No baseline yet — record one and treat as healthy (first contact).
snapshot()
print("no prior snapshot; baseline recorded")
return 0
alerts = []
now_blocks = {label: read_block(label) for label in BLOCKS}
prev = snap.get("blocks", {})
for label in BLOCKS:
cur = now_blocks[label]
cur_size = len(cur)
anchor_meta = ledger.get("anchors", {}).get(label)
if anchor_meta:
# An anchored block is measured against its authorised state — the
# re-anchor after legitimate change becomes the new reference.
base = anchor_meta.get("size", cur_size)
base_sha = anchor_meta.get("sha")
delta = cur_size - base
# Root-cause: diff current content against the anchored content.
ref_content = anchor_meta.get("content")
else:
prev_meta = prev.get(label, {})
prev_size = prev_meta.get("size", cur_size)
base = max(prev_size, 1)
base_sha = None
delta = cur_size - prev_size
ref_content = prev_meta.get("content")
pct = abs(delta) / max(base, 1)
limit = CONSTITUTION_LIMIT if label == "constitution" else DRIFT_LIMIT
if pct > limit:
alerts.append({
"label": label,
"delta": delta,
"pct": round(pct, 3),
"limit": round(limit, 3),
"base": base,
"changed": diff_blocks(ref_content, cur),
})
if alerts:
for a in alerts:
msg = (f"{a['label']}: {a['delta']:+d} bytes "
f"({a['pct']*100:.1f}% vs limit {a['limit']*100:.1f}%) "
f"[{a['changed']}]")
print(f"DRIFT-ALERT {msg}")
# AUTOMATIC FIX on detection (user directive): the system fixes
# drift itself, not just flags it.
if autofix:
auto_result = auto_fix(a["label"], ledger, strict)
print(f" auto-fix: {auto_result['action']} — {auto_result['reason']}")
if strict:
journal(f"{msg}; auto-fix: {auto_result['action']} — "
f"{auto_result['reason']}")
else:
print(f" (autofix disabled — observe only)")
return 1
print("no drift flagged (within limits)")
return 0
def auto_fix(label, ledger, strict):
"""AUTOMATIC retroactive fix upon drift detection.
Policy:
- constitution: read-only, changes ONLY by explicit directive -> never
auto-fix, alert + journal for human review.
- anchored block (authorised): the growth is legitimate -> auto re-anchor
to the new state (the fix is approving it), so future drift is measured
from here. This prevents the SAME change being flagged repeatedly.
- unanchored block: drift is unauthorised -> restore to last-good content
(the retroactive fix), removing whatever caused it.
Returns {action, reason}.
"""
if label == "constitution":
return {"action": "alert-only",
"reason": "constitution changes only by explicit directive; "
"needs human review"}
if label in ledger.get("anchors", {}):
# Authorised growth: re-anchor to the current (approved) state.
raw = read_block(label)
ledger.setdefault("anchors", {})[label] = {
"sha": _sha(raw), "size": len(raw), "content": raw,
"reason": "auto re-anchor after authorised growth",
"when": datetime.now(timezone.utc).isoformat(timespec="seconds"),
}
save_ledger(ledger)
return {"action": "re-anchored",
"reason": "anchored block grew (authorised) — re-baselined"}
# Unauthorised drift: restore to last-good content (new schema — restore
# the store's always-on core entries for this topic).
block_meta = ledger.get("last_snapshot", {}).get("blocks", {}).get(label)
if block_meta and "content" in block_meta:
restore_topic(label, block_meta["content"])
return {"action": "restored",
"reason": "unauthorised drift — restored to last-good content"}
return {"action": "alert-only",
"reason": "no last-good content available to restore"}
def diff_blocks(ref, cur):
"""Root-cause: what changed between the last-good content and now.
Returns a short list of added/removed lines (the drift's cause)."""
if not ref:
return "no prior content (first drift from empty baseline)"
try:
ref_lines = ref.splitlines()
cur_lines = cur.splitlines()
ref_set = {l.strip() for l in ref_lines if l.strip()}
cur_set = {l.strip() for l in cur_lines if l.strip()}
added = [l for l in cur_lines if l.strip() and l.strip() not in ref_set][:6]
removed = [l for l in ref_lines if l.strip() and l.strip() not in cur_set][:3]
parts = []
if added:
parts.append("ADDED: " + " | ".join(l.strip()[:60] for l in added))
if removed:
parts.append("REMOVED: " + " | ".join(l.strip()[:60] for l in removed))
return "; ".join(parts) if parts else "content changed (unclear from diff)"
except Exception:
return "diff unavailable"
def restore_topic(label, content):
"""Restore a topic's store entries from serialized content (new schema).
Replaces the always-on entries for `label` with those parsed from
`content` (the body format `read_block` produces: `- text` lines)."""
try:
import sqlite3
from datetime import datetime, timezone
db = sqlite3.connect(DB)
db.execute("DELETE FROM entries WHERE topic=? AND always_on=1 "
"AND COALESCE(method, '') != 'directive'", (label,))
ts = datetime.now(timezone.utc).isoformat(timespec="seconds")
priority = {"constitution": 0, "safety": 0, "identity": 1,
"operating": 2, "human": 3, "project": 3}.get(label, 5)
for line in content.splitlines():
text = line.strip().lstrip("-* ").strip()
if not text or len(text) < 5:
continue
db.execute(
"INSERT INTO entries (text, importance, created_at, "
"last_accessed, topic, source, method, status, valid_from, "
"confidence, temperature, always_on, priority) "
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
(text, 0.9, ts, ts, label, "drift-restore", "curator",
"active", ts, 0.9, 1.0, 1, priority))
db.commit()
db.close()
except Exception:
pass
def fix(label):
"""RETROACTIVE FIX (#B): restore a drifted block to its last-good content.
Only restores when the drift exceeds the limit AND the last-good content is
available. If the drift was legitimate (authorised growth), `anchor` instead
— fix() refuses to silently revert an anchored block.
Returns {fixed: bool, reason}.
"""
ledger = load_ledger()
snap = ledger.get("last_snapshot", {})
block_meta = snap.get("blocks", {}).get(label)
if not block_meta or "content" not in block_meta:
return {"fixed": False, "reason": f"no last-good content for {label}"}
# Never silently revert an anchored block — it was authorised.
if label in ledger.get("anchors", {}):
return {"fixed": False,
"reason": f"{label} is anchored (authorised); use anchor to "
f"re-baseline instead of fix"}
good = block_meta["content"]
cur = read_block(label)
if _sha(cur) == block_meta["sha"]:
return {"fixed": False, "reason": f"{label} already matches last-good"}
# New schema: restore the store's always-on entries for the topic (the old
# markdown block files are retired).
restore_topic(label, good)
journal(f"{label} restored to last-good content (retroactive drift fix)")
return {"fixed": True, "reason": f"{label} restored to last-good state"}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("cmd", choices=["snapshot", "check", "anchor", "fix"])
ap.add_argument("label", nargs="?", default=None,
help="block label for anchor")
ap.add_argument("reason", nargs="?", default="", help="anchor reason")
ap.add_argument("--strict", action="store_true",
help="journal DRIFT-ALERT on violation")
ap.add_argument("--no-autofix", action="store_true",
help="check only — don't apply automatic fix (observe only)")
args = ap.parse_args()
if args.cmd == "snapshot":
print(json.dumps(snapshot()))
sys.exit(0)
if args.cmd == "check":
sys.exit(check(strict=args.strict, autofix=not args.no_autofix))
if args.cmd == "fix":
if not args.label:
print("fix needs <label>", file=sys.stderr)
sys.exit(1)
print(json.dumps(fix(args.label)))
sys.exit(0)
if args.cmd == "anchor":
if not args.label:
print("anchor needs <label>", file=sys.stderr)
sys.exit(1)
print(anchor(args.label, args.reason or "authorised by the user"))
sys.exit(0)
if __name__ == "__main__":
main()
+229
View File
@@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""lexicon_evolution.py — the lexicon meta-judgement applied to EVOLUTION.
Retrieval already applies the meta-judgement ("a shared word is not a
relation; a shared DISTINCTIVE word is a signal"): conjunctive matching, rare-
word gating, calibrated floors. Evolution decisions (persona coherence, identity
grounding, constitution promotion) must use the SAME lexicon discipline —
otherwise a personality can "evolve" on a coincidental shared word exactly the
way recall used to misfire on "black coffee" vs "black holes".
This module provides the shared lexical signal for the curator's three growth
paths, using identical definitions to the recall gate (same stopword set, same
distinctiveness threshold), so growth and retrieval speak one lexicon.
Decisions:
coherence(fact, existing_values) -> "consistent" | "conflict" | "novel"
Does a new persona/identity value COHERE with what already exists?
- "conflict": shares a DISTINCTIVE term AND a negation marker on it
(a real contradiction, e.g. "never jokes" vs "dry humor").
- "consistent": shares a distinctive term without negation (reinforces).
- "novel": shares nothing distinctive — a new, additive axis (still
promotable, but it does NOT rest on a shared-word coincidence).
A shared COMMON word ("work", "always", "way") is never a relation.
grounded(fact, source_texts) -> bool
Does the fact trace to real absorbed material via a DISTINCTIVE shared
term (conjunctive), not just vector proximity or a common word?
Lexical twin of the recall gate's dense check.
promote_eligible(proposal, existing, evidence_strength, min_strength) -> dict
A constitution/persona promotion is eligible only when BOTH hold:
- the evidence threshold is met (existing durability gate), AND
- the proposal shares a DISTINCTIVE term with the existing block OR is
a fully novel axis (a real new principle, not a word-coincidence).
Returns {eligible, reason}.
Usage:
from lexicon_evolution import coherence, grounded, promote_eligible
"""
import os
import re
# SAME stopword set as memory_store._STOPWORDS (single source of truth would be
# ideal; keep in sync — see memory_store.py).
_STOP = {"the", "and", "for", "with", "that", "this", "from", "into",
"when", "then", "were", "have", "been", "will", "was", "are",
"but", "not", "you", "your", "also", "its", "his", "her", "him",
"over", "under", "them", "they", "there", "about", "after",
"what", "which", "who", "how", "why", "where", "while", "just",
"more", "each", "than", "then", "very", "such", "some", "only",
"prefers", "needs", "wants", "likes"}
# Hard stop for evolution: these generic framing words never carry a relation
# ("work", "always", "way") — a persona built on them would be a coincidence.
_FRAMING = {"work", "always", "way", "thing", "value", "identity", "persona",
"someone", "something", "never", "always",
"before", "after", "then", "when", "while", "must", "should",
"will", "would", "could", "can", "have", "has", "had", "been",
"being", "used", "use", "uses", "make", "makes", "made", "keep",
"keeps", "take", "takes", "took", "get", "gets", "got", "going"}
_NEGATION = {"not", "never", "don't", "doesn't", "cannot", "can't", "no",
"nor", "neither", "without", "avoid", "reject", "against",
"refuses", "refuse", "opposed"}
_DISTINCTIVE_DF = 0.05 # same rare-word threshold as recall/associations
def content_words(text):
"""Significant content words (length > 3, not stop/framing)."""
return [w for w in re.findall(r"[a-z]{4,}", (text or "").lower())
if w not in _STOP and w not in _FRAMING]
def distinctive_terms(fact, corpus_texts):
"""The terms in `fact` that are RARE across the corpus (< 5% df).
Same signal as the recall gate: a term common in the store ("black",
"physics") is NOT a relation; a rare term ("celesta", "oatmilk") is."""
total = max(len(corpus_texts), 1)
fact_terms = set(content_words(fact))
if not fact_terms:
return set()
# stem-aware document frequency (migrate/migration count as one term)
doc_terms = {}
for t in corpus_texts:
stems = {_stem(w) for w in re.findall(r"[a-z]{4,}", (t or "").lower())}
for s in stems:
doc_terms[s] = doc_terms.get(s, 0) + 1
return {t for t in fact_terms
if doc_terms.get(_stem(t), 0) < max(total * _DISTINCTIVE_DF, 1.5)}
def shared_distinctive(fact, existing_values, corpus_texts):
"""Distinctive terms shared between `fact` and existing values."""
distinct = distinctive_terms(fact, corpus_texts)
if not distinct:
return set()
shared = set()
existing_stems = [(_stem(w), val) for val in existing_values
for w in re.findall(r"[a-z]{4,}", (val or "").lower())]
for term in distinct:
ts = _stem(term)
for s, _val in existing_stems:
if s == ts:
shared.add(term)
break
return shared
def coherence(fact, existing_values, corpus_texts=()):
"""Classify a new value against existing ones, lexicon-aware.
Returns "conflict" | "consistent" | "novel".
"""
values = [v for v in existing_values if v and v.strip()]
if not values:
return "novel"
corpus = corpus_texts or values
shared = shared_distinctive(fact, values, corpus)
if not shared:
# No distinctive shared term: the value is a NEW axis. This is NOT a
# contradiction (a shared common word like "work" is meaningless), and
# NOT a reinforcement — it adds independently.
return "novel"
low = fact.lower()
has_neg = any(n in low for n in _NEGATION)
if has_neg and any(f" {t} " in f" {low} " for t in shared):
return "conflict"
return "consistent"
def grounded(fact, source_texts, min_distinct=1):
"""Is `fact` grounded in real material via a DISTINCTIVE shared term?
Conjunctive/rare-word check (lexical twin of the recall gate): a fact that
shares only common words or pure vector proximity with sources is NOT
grounded — invented concepts must not shape personality/identity.
Morphology-aware: a term is grounded if its STEM appears in the sources
("migrated" matches "migration", "retroactively" matches "retroactive"),
using a light suffix fold — exact word identity is too strict for natural
sources.
"""
srcs = [s for s in source_texts if s and s.strip()]
if not srcs:
return False
fact_terms = set(content_words(fact))
if not fact_terms:
return False
joined = " ".join(s.lower() for s in srcs)
stems = {_stem(t) for t in re.findall(r"[a-z]{4,}", joined)}
hits = 0
for t in fact_terms:
if _stem(t) in stems:
hits += 1
if hits >= min_distinct:
return True
return False
def _stem(w):
"""Light morphological fold for grounding: drop common English suffixes.
Enough to equate migrate/migration/retroactively/retroactive and
honest/honesty without a full stemmer (no external deps).
The suffix rules are ordered longest-first and grouped so that the 5-char
participle forms ('ating', 'ated') are stripped before their 3-char base
('ing', 'ed'), keeping migrate/migrating/migrated/migration on one root.
"""
# (suffix, optional extra fold applied after stripping)
rules = [
("ationally", ""), ("ableness", ""), ("iveness", ""),
("ation", ""), ("ating", ""), ("ative", "e"),
("atively", "e"), ("ability", ""), ("ities", "y"),
("ingly", ""), ("fully", ""), ("ness", ""), ("ment", ""),
("itive", "e"), ("atable", "e"), ("ated", ""), ("ates", "e"),
("ions", ""), ("ings", ""), ("able", "e"), ("ible", "e"),
("ing", "e"), ("ed", ""), ("es", ""), ("ly", ""), ("s", ""), ("e", ""),
]
orig = w
for suf, fold in rules:
if len(w) - len(suf) >= 4 and w.endswith(suf):
w = w[: len(w) - len(suf)]
if fold:
w = w + fold
break
# y -> i fold (honesty -> honest)
if len(w) > 4 and w.endswith("y") and w[-2] not in "aeiouy":
w = w[:-1]
return w
def promote_eligible(proposal, existing, evidence_strength, min_strength=3,
corpus_texts=()):
"""Is a persona/constitution promotion eligible, lexicon-aware?
Eligibility = evidence threshold AND (shares a distinctive term with the
existing block OR is a genuinely novel axis grounded in the corpus). A
proposal that rests only on common words is a word-coincidence and is
parked, no matter how often it is sighted — mirroring the durability gate's
"3+ references" rule but with a lexical floor. A "novel axis" is only
eligible when it carries a DISTINCTIVE term that actually appears in the
real absorbed material (the corpus), so arbitrary trivia ("pizza on
fridays") cannot be promoted as a principle no matter the sighting count.
"""
if evidence_strength < min_strength:
return {"eligible": False,
"reason": f"evidence strength {evidence_strength} < {min_strength}"}
existing = [e for e in existing if e and e.strip()]
if not existing:
return {"eligible": True, "reason": "empty block — first principle"}
corpus = corpus_texts or existing
shared = shared_distinctive(proposal, existing, corpus)
if shared:
return {"eligible": True,
"reason": f"shares distinctive term(s): {sorted(shared)[:3]}"}
# No distinctive shared term: a NOVEL axis is eligible only if it is
# GROUNDED in the absorbed corpus via a distinctive term (a real principle
# names something real), and the proposal is not trivial framing.
terms = content_words(proposal)
if not terms:
return {"eligible": False, "reason": "no lexical signal"}
if grounded(proposal, corpus, min_distinct=1):
return {"eligible": True,
"reason": f"novel axis grounded in corpus ({len(terms)} terms)"}
return {"eligible": False,
"reason": "novel axis not grounded in absorbed material — parked"}
File diff suppressed because it is too large. Load diff