Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Nick
|
c12c155dd8 | ||
|
Nick
|
c01d33f6e8 | ||
|
Nick
|
edc20b9e42 | ||
|
Nick
|
a3ae72d353 | ||
|
Nick
|
dc99f1fac3 |
No files matched your search
@@ -8,7 +8,7 @@ on:
|
||||
|
||||
jobs:
|
||||
backup:
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
# actions/checkout v4.2.2, pinned to the audited upstream tag commit.
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
|
||||
@@ -12,7 +12,7 @@ jobs:
|
||||
# a demand comment — filing is not complete until prioritized.
|
||||
priority-gate:
|
||||
if: github.event_name == 'issues' && github.event.action != 'labeled'
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- name: Enforce priority label
|
||||
uses: actions/github-script@v7
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
});
|
||||
|
||||
auto-label:
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- name: Analyze and label
|
||||
uses: actions/github-script@v7
|
||||
|
||||
@@ -8,7 +8,7 @@ on:
|
||||
|
||||
jobs:
|
||||
update-gitea:
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- name: Check current Gitea version
|
||||
id: current
|
||||
|
||||
@@ -9,7 +9,7 @@ permissions: write-all
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -29,152 +29,20 @@ jobs:
|
||||
echo "RELEASE_VERSION=$VERSION" >> "$GITHUB_ENV"
|
||||
echo "Release version: $VERSION"
|
||||
|
||||
- name: Auto-bump if the PR did not carry a version bump
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
RELEASE_SERVER_URL: ${{ gitea.server_url }}
|
||||
RELEASE_REPO: ${{ gitea.repository }}
|
||||
PR_TITLE: ${{ gitea.event.pull_request.title }}
|
||||
PR_BODY: ${{ gitea.event.pull_request.body }}
|
||||
- name: Verify version was bumped since last tag
|
||||
run: |
|
||||
LATEST=$(git tag --sort=-v:refname | head -1 | sed 's/^v//')
|
||||
if [ -z "$LATEST" ]; then
|
||||
echo "No previous tags — releasing first version $RELEASE_VERSION"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$LATEST" != "$RELEASE_VERSION" ]; then
|
||||
echo "New version $RELEASE_VERSION (last tag v$LATEST) — bump rode the PR"
|
||||
exit 0
|
||||
if [ "$LATEST" = "$RELEASE_VERSION" ]; then
|
||||
echo "::error::Version $RELEASE_VERSION was already released (tag v$LATEST)."
|
||||
echo "::error::Semver bump missing from this PR: add a 'chore(release): bump version to X.Y.Z' commit that increments src/rune/__init__.py, then merge again."
|
||||
exit 1
|
||||
fi
|
||||
NEW=$(python3 -c "
|
||||
import os, re
|
||||
version = os.environ['RELEASE_VERSION']
|
||||
title = os.environ.get('PR_TITLE', '')
|
||||
# Conventional-commits semver: feat! / BREAKING CHANGE -> major, feat -> minor, else patch
|
||||
if re.match(r'^\s*\w+(\([^)]*\))?!:', title) or 'BREAKING CHANGE' in title:
|
||||
kind = 'major'
|
||||
elif re.match(r'^\s*feat(\([^)]*\))?:', title):
|
||||
kind = 'minor'
|
||||
else:
|
||||
kind = 'patch'
|
||||
major, minor, patch = version.split('.')
|
||||
# 0.x semver convention: breaking changes bump the MINOR until 1.0.0
|
||||
result = (
|
||||
f'{major}.{int(minor)+1}.0' if kind == 'major' and int(major) == 0
|
||||
else f'{int(major)+1}.0.0' if kind == 'major'
|
||||
else f'{major}.{int(minor)+1}.0' if kind == 'minor'
|
||||
else f'{major}.{minor}.{int(patch)+1}'
|
||||
)
|
||||
print(result)
|
||||
")
|
||||
export NEW_VERSION="$NEW"
|
||||
echo "Auto-bump: $RELEASE_VERSION -> $NEW (via protected-branch-safe bump PR)"
|
||||
python3 - <<'PYBUMP'
|
||||
import json, os, sys, time, urllib.request
|
||||
echo "New version $RELEASE_VERSION (last tag v$LATEST)"
|
||||
|
||||
TOKEN = os.environ["GITEA_TOKEN"]
|
||||
BASE = os.environ["RELEASE_SERVER_URL"] + "/api/v1/repos/" + os.environ["RELEASE_REPO"]
|
||||
VERSION = os.environ["RELEASE_VERSION"]
|
||||
NEW = os.environ["NEW_VERSION"]
|
||||
PR_TITLE = os.environ.get("PR_TITLE", "")
|
||||
PR_BODY = os.environ.get("PR_BODY", "")
|
||||
BRANCH = f"chore/auto-bump-{NEW}"
|
||||
|
||||
def api(path, method="GET", data=None, ok=(200,)):
|
||||
req = urllib.request.Request(
|
||||
BASE + path,
|
||||
data=json.dumps(data).encode() if data is not None else None,
|
||||
headers={"Authorization": "token " + TOKEN, "Content-Type": "application/json"},
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
body = r.read()
|
||||
return r.status, (json.loads(body) if body else {})
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", "replace")
|
||||
print(f" API {method} {path} -> {e.code}: {body[:200]}")
|
||||
return e.code, {"_body": body}
|
||||
|
||||
# 1. bump branch from master
|
||||
status, _ = api("/branches", "POST", {"new_branch_name": BRANCH, "old_branch_name": "master"})
|
||||
if status not in (201, 409):
|
||||
print(f"branch create failed ({status})"); sys.exit(1)
|
||||
|
||||
# 2. commit the five version-surface edits via the contents API
|
||||
surfaces = {
|
||||
"src/rune/__init__.py": [
|
||||
(f'__version__ = "{VERSION}"', f'__version__ = "{NEW}"'),
|
||||
],
|
||||
"src/rune/rune_tui.py": [
|
||||
(f"Version: v{VERSION}", f"Version: v{NEW}"),
|
||||
(f"Rune v{VERSION} |", f"Rune v{NEW} |"),
|
||||
],
|
||||
"src/rune/canary_gitea_automation.py": [
|
||||
(f'"{VERSION}"', f'"{NEW}"'),
|
||||
],
|
||||
"README.md": [
|
||||
(f"v{VERSION}", f"v{NEW}"),
|
||||
],
|
||||
}
|
||||
for path, pairs in surfaces.items():
|
||||
status, cur = api(f"/contents/{path}?ref={BRANCH}")
|
||||
if status != 200:
|
||||
# optional surface (e.g. pattern absent) — skip gracefully
|
||||
print(f" skip {path} ({status})"); continue
|
||||
import base64
|
||||
text = base64.b64decode(cur["content"]).decode("utf-8")
|
||||
changed = text
|
||||
for old, newv in pairs:
|
||||
changed = changed.replace(old, newv)
|
||||
if changed == text:
|
||||
print(f" no-op {path}"); continue
|
||||
status, r = api(f"/contents/{path}", "PUT", {
|
||||
"message": f"chore(release): auto-bump version to {NEW} (post-merge automation)",
|
||||
"content": base64.b64encode(changed.encode()).decode(),
|
||||
"branch": BRANCH, "sha": cur["sha"],
|
||||
})
|
||||
if status not in (200, 201):
|
||||
print(f" commit {path} failed ({status})"); sys.exit(1)
|
||||
|
||||
# 3. bump PR
|
||||
status, pr = api("/pulls", "POST", {
|
||||
"title": f"chore(release): auto-bump version to {NEW} (post-merge automation)",
|
||||
"body": f"Automatic version bump after merge of: {PR_TITLE}\n\n{PR_BODY[:1500]}",
|
||||
"head": BRANCH, "base": "master",
|
||||
})
|
||||
if status not in (200, 201):
|
||||
print(f"PR create failed ({status})"); sys.exit(1)
|
||||
pr_number = pr["number"]
|
||||
head_sha = pr["head"]["sha"]
|
||||
print(f" bump PR #{pr_number} head {head_sha[:12]}")
|
||||
|
||||
# 4. wait for the protected check on the bump PR (same gate as everything else)
|
||||
deadline = time.time() + 900 # 15 min ceiling
|
||||
state = None
|
||||
while time.time() < deadline:
|
||||
st, cs = api(f"/commits/{head_sha}/status")
|
||||
state = cs.get("state")
|
||||
if state in ("success", "failure"):
|
||||
break
|
||||
time.sleep(15)
|
||||
if state != "success":
|
||||
print(f"bump PR CI did not go green (state={state})"); sys.exit(1)
|
||||
|
||||
# 5. merge via API (respects branch protection; user_can_merge)
|
||||
status, _ = api(f"/pulls/{pr_number}/merge", "POST", {"Do": "merge"})
|
||||
if status not in (200, 405):
|
||||
print(f"merge failed ({status})"); sys.exit(1)
|
||||
|
||||
# 6. confirm merged, then export the version for the release step
|
||||
st, pr2 = api(f"/pulls/{pr_number}")
|
||||
if not pr2.get("merged"):
|
||||
print("merge did not take"); sys.exit(1)
|
||||
print(f"AUTO_BUMPED=true")
|
||||
with open(os.environ["GITHUB_ENV"], "a") as env:
|
||||
env.write(f"RELEASE_VERSION={NEW}\n")
|
||||
env.write("AUTO_BUMPED=true\n")
|
||||
PYBUMP
|
||||
- name: Create release from merged pull request
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
|
||||
+8
-12
@@ -4,11 +4,10 @@ on:
|
||||
push:
|
||||
pull_request:
|
||||
branches: [master]
|
||||
workflow_dispatch: # benchmark: run representative CI without manufacturing commits
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
# actions/checkout v4.2.2, pinned to the audited upstream tag commit.
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
@@ -42,7 +41,7 @@ jobs:
|
||||
.ci-venv/bin/henxels check --all --plain
|
||||
|
||||
unit-tests:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
# actions/checkout v4.2.2, pinned to the audited upstream tag commit.
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
@@ -55,7 +54,7 @@ jobs:
|
||||
PYTHONPATH=src:. .test-venv/bin/python -m pytest tests/ -q
|
||||
|
||||
canary:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
env:
|
||||
RUNE_HOME: /tmp/rune-ci-${{ github.run_id }}
|
||||
RUNE_MEMORY_DIR: /tmp/rune-ci-${{ github.run_id }}/memory
|
||||
@@ -90,10 +89,7 @@ jobs:
|
||||
src/rune/canary_p0_04.py \
|
||||
src/rune/canary_p0_05.py \
|
||||
src/rune/canary_p0_07.py \
|
||||
src/rune/canary_p1_02.py \
|
||||
src/rune/canary_p1_07.py \
|
||||
src/rune/canary_p1_06.py \
|
||||
src/rune/canary_p1_01.py \
|
||||
src/rune/canary_query_memory.py \
|
||||
src/rune/canary_unbounded.py \
|
||||
src/rune/canary_full_system.py \
|
||||
@@ -120,7 +116,7 @@ jobs:
|
||||
esac
|
||||
|
||||
best-practices:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
# actions/checkout v4.2.2, pinned to the audited upstream tag commit.
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
@@ -144,7 +140,7 @@ jobs:
|
||||
echo " $COUNT TODO/FIXME items found"
|
||||
|
||||
documentation:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
# actions/checkout v4.2.2, pinned to the audited upstream tag commit.
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
@@ -175,7 +171,7 @@ jobs:
|
||||
"
|
||||
|
||||
type-check:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
- name: Create type-check environment from manifest
|
||||
@@ -187,7 +183,7 @@ jobs:
|
||||
.type-venv/bin/python scripts/type_check_ratchet.py
|
||||
|
||||
gitea-offline:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
# actions/checkout v4.2.2, pinned to the audited upstream tag commit.
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
|
||||
@@ -199,7 +195,7 @@ jobs:
|
||||
|
||||
required-checks:
|
||||
if: gitea.event_name == 'pull_request'
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
# #148: type-check MUST gate the protected status — a red type-check job
|
||||
# previously left the protected merge gate green (dependency-chain hole).
|
||||
needs: [lint, unit-tests, canary, best-practices, documentation, type-check, gitea-offline]
|
||||
|
||||
@@ -8,7 +8,7 @@ on:
|
||||
|
||||
jobs:
|
||||
lint-and-format:
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Lint Python files
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
ruff format shared/ src/ || true
|
||||
|
||||
canary-suite:
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Run all canaries
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
exit $FAILED
|
||||
|
||||
best-practices:
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Check best practices
|
||||
@@ -67,7 +67,7 @@ jobs:
|
||||
grep -rn "TODO\|FIXME\|HACK\|XXX" shared/ src/ --include="*.py" | head -10 || echo " Clean"
|
||||
|
||||
documentation-check:
|
||||
runs-on: rune-infra-pi
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Check documentation coverage
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
name: Merge robot
|
||||
|
||||
# Deterministic merge-on-green for this repo's PRs. No LLM, no polling by the
|
||||
# agent: the forge event model does the watching, this job does the one action.
|
||||
# Runs hourly + on CI completion events so green PRs merge without any agent
|
||||
# turn being spent on "check CI, merge, clean up".
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '23 * * * *'
|
||||
workflow_dispatch: {}
|
||||
workflow_run:
|
||||
workflows: ["CI Pipeline", "Auto-version and Release"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
pull_requests: write
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
env:
|
||||
OWNER: conductor
|
||||
REPO: rune
|
||||
|
||||
jobs:
|
||||
merge-green:
|
||||
runs-on: rune-ci-desktop
|
||||
steps:
|
||||
- name: Merge PRs whose protected checks are green (fail-loud)
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
SERVER: ${{ gitea.server_url }}
|
||||
run: |
|
||||
python3 scripts/merge_robot.py "$GITEA_TOKEN" "$SERVER/api/v1/repos/conductor/rune"
|
||||
@@ -3,8 +3,6 @@ name: Pipeline Order Gate
|
||||
on:
|
||||
issues:
|
||||
types: [labeled, closed]
|
||||
issue_comment:
|
||||
types: [created]
|
||||
schedule:
|
||||
# Hourly sweep: catches newly-eligible issues when a dependency closed
|
||||
# while the teacher was down/not yet deployed (temporal gap fix).
|
||||
@@ -18,7 +16,7 @@ jobs:
|
||||
# `Depends on: #N` reference in its body is CLOSED. Otherwise Gitea
|
||||
# blocks the stage (removes ready, adds blocked, names the blocker).
|
||||
gate:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- name: Verify dependency order before allowing ready
|
||||
uses: actions/github-script@v7
|
||||
@@ -60,15 +58,9 @@ jobs:
|
||||
}
|
||||
}
|
||||
if (openDeps.length === 0) {
|
||||
// Auto-apply research-pending: the issue cannot start until a
|
||||
// 4-phase research round (INWARD / OUTWARD / INNOVATION / SYNTHESIS)
|
||||
// is completed and posted as a comment.
|
||||
await github.rest.issues.addLabels({
|
||||
owner, repo, issue_number: issue.number, labels: ['research-pending'],
|
||||
}).catch(e => { if (e.status !== 404) throw e; });
|
||||
await github.rest.issues.createComment({
|
||||
owner, repo, issue_number: issue.number,
|
||||
body: `🟢 **READY — order satisfied.** Dependencies: ${uniqueDeps.length ? uniqueDeps.map(n => '#' + n).join(', ') : 'none'}.\n\n📋 **Research round required** (RESEARCH ROUND V3 — 4 phases):\n1. **INWARD** — codebase analysis\n2. **OUTWARD** — state-of-the-art survey (libraries, papers, production systems; "is there anything BETTER?")\n3. **INNOVATION** — viability-filtered improvements (evaluate, optionally implement)\n4. **SYNTHESIS** — specific build instructions\n\nRune (the agent) performs the research and posts the findings as a comment on this issue. The \`research-pending\` label will be cleared automatically and the issue becomes startable.`,
|
||||
body: `🟢 **READY — order satisfied; Rune may start.** Dependencies: ${uniqueDeps.length ? uniqueDeps.map(n => '#' + n).join(', ') : 'none'}.`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
@@ -90,7 +82,7 @@ jobs:
|
||||
# issue (open, has the orchestration-stage marker in its body, all deps
|
||||
# closed) and label it ready — the order teacher keeps Rune on task.
|
||||
teach:
|
||||
runs-on: rune-ci-desktop
|
||||
runs-on: self-hosted
|
||||
steps:
|
||||
- name: Announce the next issue in dependency order
|
||||
uses: actions/github-script@v7
|
||||
@@ -162,303 +154,3 @@ jobs:
|
||||
body: (() => { const closed = context.payload.issue ? ' (closed: #' + context.payload.issue.number + ')' : ' (scheduled sweep)'; return `🎓 **NEXT: #${next.number} — ${next.title}** — dependencies satisfied${closed}; Rune may start.`; })(),
|
||||
});
|
||||
core.info(`Announced #${next.number} as next.`);
|
||||
|
||||
# RESEARCH CLEAR: when a comment containing the 4-phase research markers
|
||||
# lands on a research-pending issue, clear the label and announce start.
|
||||
research-clear:
|
||||
runs-on: rune-ci-desktop
|
||||
if: github.event_name == 'issue_comment' && github.event.action == 'created'
|
||||
steps:
|
||||
- name: Clear research-pending when research round is posted
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const issue = context.payload.issue;
|
||||
if (!issue) { core.info('no issue in payload'); return; }
|
||||
const comment = context.payload.comment?.body || '';
|
||||
|
||||
// --- Phase header detection (case-insensitive) ---
|
||||
const hasInward = /\bINWARD\b/i.test(comment);
|
||||
const hasOutward = /\bOUTWARD\b/i.test(comment);
|
||||
const hasInnovation = /\bINNOVATION\b/i.test(comment);
|
||||
const hasSynthesis = /\bSYNTHESIS\b/i.test(comment);
|
||||
if (!hasInward || !hasOutward || !hasInnovation || !hasSynthesis) {
|
||||
core.info(`research-clear: comment on #${issue.number} missing phases (inward=${hasInward} outward=${hasOutward} innovation=${hasInnovation} synthesis=${hasSynthesis}) — skip`);
|
||||
return;
|
||||
}
|
||||
|
||||
// --- Evidence enforcement (per phase, section-scoped) ---
|
||||
// Split the comment into phase sections on the phase headers.
|
||||
function section(phase) {
|
||||
const re = new RegExp(`\\b${phase}\\b[\\s\\S]*?(?=\\b(?:INWARD|OUTWARD|INNOVATION|SYNTHESIS)\\b|$)`, 'i');
|
||||
const m = comment.match(re);
|
||||
return m ? m[0] : '';
|
||||
}
|
||||
const forgeHost = context.payload.repository?.html_url ? new URL(context.payload.repository.html_url).host : '';
|
||||
const externalUrlRe = /https?:\/\/[^\s)\]>]+/gi;
|
||||
const codePathRe = /(?:^|\s)(?:src|tests|scripts|docs|\.gitea)\/[\w./-]+/gi;
|
||||
const plansPathRe = /(?:^|\s)plans\/[\w./-]+/gi;
|
||||
|
||||
function uniqueExternalUrls(text) {
|
||||
const found = text.match(externalUrlRe) || [];
|
||||
const uniq = new Map(); // normalized key -> original URL (fetchable)
|
||||
for (const u of found) {
|
||||
try {
|
||||
const parsed = new URL(u);
|
||||
if (parsed.host && parsed.host !== forgeHost) {
|
||||
const key = parsed.host + parsed.pathname;
|
||||
if (!uniq.has(key)) uniq.set(key, u);
|
||||
}
|
||||
} catch (e) { /* skip malformed */ }
|
||||
}
|
||||
return uniq;
|
||||
}
|
||||
function codeCitations(text) {
|
||||
return new Set((text.match(codePathRe) || []).map(s => s.trim()));
|
||||
}
|
||||
|
||||
const inwardSec = section('INWARD');
|
||||
const outwardSec = section('OUTWARD');
|
||||
const innovationSec = section('INNOVATION');
|
||||
const synthesisSec = section('SYNTHESIS');
|
||||
const tailSec = innovationSec + '\n' + synthesisSec;
|
||||
|
||||
const inwardCites = codeCitations(inwardSec);
|
||||
const outwardUrls = uniqueExternalUrls(outwardSec);
|
||||
const tailUrls = uniqueExternalUrls(tailSec);
|
||||
const tailPlans = (tailSec.match(plansPathRe) || []).length;
|
||||
|
||||
const missing = [];
|
||||
if (inwardCites.size < 2) missing.push(`- **INWARD**: needs ≥2 codebase citations (paths like src/..., tests/..., .gitea/...) — found ${inwardCites.size}.`);
|
||||
if (outwardUrls.size < 2) missing.push(`- **OUTWARD**: needs ≥2 UNIQUE external source URLs (not this forge) — found ${outwardUrls.size}. Research must pull real sources to learn from so later stages build on evidence.`);
|
||||
if (tailUrls.size < 1 && tailPlans < 1) missing.push(`- **INNOVATION/SYNTHESIS**: needs ≥1 source URL reference or a plans/ artifact path tying synthesis back to evidence.`);
|
||||
|
||||
// --- INNOVATION must be source-tied: ≥1 external URL from OUTWARD
|
||||
// re-referenced (proving candidates derive from pulled sources, not invention).
|
||||
const innovationUrlKeys = [...uniqueExternalUrls(innovationSec).keys()];
|
||||
const tiedToOutward = innovationUrlKeys.filter(k => outwardUrls.has(k));
|
||||
if (outwardUrls.size >= 2 && tiedToOutward.length < 1) {
|
||||
missing.push(`- **INNOVATION**: candidate approaches must derive from OUTWARD sources — re-reference ≥1 of the OUTWARD source URLs here and say what it contributed. Found 0 source-tied candidates.`);
|
||||
}
|
||||
|
||||
// --- SYNTHESIS qualification gate: the synthesis must (a) declare its
|
||||
// evidence sufficiency, (b) judge feasibility REALISTICALLY (mechanism +
|
||||
// assumptions + failure modes), and (c) either commit to a plan artifact
|
||||
// or honestly bounce to MORE RESEARCH. Evidence-insufficient synthesis
|
||||
// MUST declare MORE RESEARCH — bluffing through fails the gate.
|
||||
const SYNTH_REJECTS = [];
|
||||
const suffMarker = /\bevidence[^\n]{0,40}(suffic|adequat|insuffic)|insufficient evidence|evidence gap/i;
|
||||
const hasSufficiency = suffMarker.test(synthesisSec);
|
||||
const moreResearch = /\bMORE[- ]RESEARCH\b/i.test(synthesisSec);
|
||||
const hasPlanArtifact = /(?:^|\s)plans\/[\w./-]+/.test(synthesisSec);
|
||||
if (moreResearch) {
|
||||
// Honest loop-back path: open questions REQUIRED, feasibility verdict
|
||||
// explicitly NOT required — a substantiated MORE-RESEARCH is the honest
|
||||
// outcome; demanding feasibility prose would teach padding.
|
||||
const openQs = (synthesisSec.match(/\?\s*$/gm) || []).length + (synthesisSec.match(/\bopen question|\bunknown|\bunresolved/gi) || []).length;
|
||||
if (openQs < 1) {
|
||||
SYNTH_REJECTS.push(`- **MORE-RESEARCH declared but unsubstantiated**: name the specific open questions/unknowns that require more research (at least one '?', 'open question', or 'unknown').`);
|
||||
}
|
||||
if (!hasSufficiency) {
|
||||
SYNTH_REJECTS.push(`- **MORE-RESEARCH qualification**: state what evidence you have, what is missing, and why the gap blocks a design decision (evidence-sufficiency statement).`);
|
||||
}
|
||||
} else {
|
||||
// Proceed-to-plan path: full qualification — sufficiency + feasibility + kill test.
|
||||
if (!hasSufficiency) {
|
||||
SYNTH_REJECTS.push(`- **SYNTHESIS qualification**: no evidence-sufficiency statement. State explicitly whether OUTWARD evidence is sufficient to design from, and why (e.g. 'Evidence sufficient: <2-3 sources converge on the mechanism>' or 'Evidence insufficient: <what specifically is missing>').`);
|
||||
}
|
||||
const feasMarker = /\bfeasib|\bwon'?t work\b|\bwill work\b|\brisk\b|\bfailure mode\b|\bassumption\b|\brealistic/i;
|
||||
const hasFeasibility = feasMarker.test(synthesisSec);
|
||||
if (!hasFeasibility) {
|
||||
SYNTH_REJECTS.push(`- **SYNTHESIS qualification**: no feasibility judgement. State the working mechanism, the load-bearing assumptions, and the failure modes — and judge realistically whether the synthesis WILL WORK. Optimism is not evidence.`);
|
||||
}
|
||||
// Kill test: a feasibility claim must name its cheapest disproof.
|
||||
// The plan must carry it as an early frozen test — prose is not a prediction.
|
||||
const hasKillTest = /\bkill test\b|\bdisprov|\bfalsif/i.test(synthesisSec);
|
||||
if (hasFeasibility && !hasKillTest) {
|
||||
SYNTH_REJECTS.push(`- **SYNTHESIS kill test**: name the cheapest observation that would DISPROVE the feasibility claim (a 'kill test:' line). For workflow-only changes the protected CI run may be the kill test — say so explicitly. A claim with no disproof is not a claim.`);
|
||||
}
|
||||
if (!hasPlanArtifact) {
|
||||
SYNTH_REJECTS.push(`- **SYNTHESIS disposition**: must reference a concrete plans/ artifact (ready to implement) or declare **MORE-RESEARCH** with the specific open questions. Neither found.`);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Trail of use: each OUTWARD source must carry an inline annotation
|
||||
// (what was learned from it) on the same line — the verifiable trail.
|
||||
const annotated = [...outwardUrls.entries()].filter(([, url]) => {
|
||||
const line = outwardSec.split('\n').find(l => l.includes(url));
|
||||
if (!line) return false;
|
||||
const prose = line.replace(externalUrlRe, '').replace(/[-*`>\[\]()]/g, '').trim();
|
||||
return prose.length >= 15;
|
||||
});
|
||||
if (outwardUrls.size >= 2 && annotated.length < 2) {
|
||||
missing.push(`- **OUTWARD trail**: each source needs an inline note of what it taught (URL + ≥15 chars of what was learned, same line) — ${annotated.length}/${outwardUrls.size} annotated. A bare URL list is not a research trail.`);
|
||||
}
|
||||
|
||||
// --- Live verification: fetch up to 5 sources; dead/hallucinated links fail.
|
||||
// 2xx/3xx = real; 403/405/406/429 = real-but-bot-blocked (count as verified);
|
||||
// 404/410/DNS/timeout (after 1 retry) = dead.
|
||||
const toVerify = [...outwardUrls.values()].slice(0, 5);
|
||||
const dead = [];
|
||||
for (const u of toVerify) {
|
||||
let ok = false;
|
||||
for (let attempt = 0; attempt < 2 && !ok; attempt++) {
|
||||
try {
|
||||
const ctl = new AbortController();
|
||||
const t = setTimeout(() => ctl.abort(), 8000);
|
||||
const r = await fetch(u, { redirect: 'follow', signal: ctl.signal, headers: { 'User-Agent': 'rune-research-gate/1.0 (source verification)' } });
|
||||
clearTimeout(t);
|
||||
ok = (r.status < 400) || [403, 405, 406, 429].includes(r.status);
|
||||
if (!ok && [404, 410].includes(r.status)) break; // definitively dead, no retry
|
||||
} catch (e) {
|
||||
clearTimeout(t);
|
||||
if (attempt === 1) { /* second failure — stays dead */ }
|
||||
}
|
||||
}
|
||||
if (!ok) dead.push(u);
|
||||
}
|
||||
if (dead.length > 0) {
|
||||
missing.push(`- **OUTWARD verification**: source(s) failed live fetch: ${dead.join(', ')} — dead or unreachable links cannot serve as research evidence.`);
|
||||
}
|
||||
|
||||
const labels0 = (issue.labels || []).map(l => l.name);
|
||||
if (!labels0.includes('research-pending')) {
|
||||
core.info(`research-clear: #${issue.number} has no research-pending label — skip`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
body: `🚫 **Research round REJECTED — evidence gate.** All 4 phase headers present, but the evidence floor is not met:\n\n${missing.join('\n')}\n\nresearch-pending stays. Post the missing evidence (real sources fetched, real paths read) and the gate will re-run on your next comment.`,
|
||||
});
|
||||
core.info(`research-clear: evidence gate rejected #${issue.number}: ${missing.length} gap(s)`);
|
||||
return;
|
||||
}
|
||||
|
||||
// --- Honest loop-back: synthesis declared MORE-RESEARCH with substantiated
|
||||
// open questions and all evidence floors met. This is a SUCCESS for honesty:
|
||||
// research-pending stays, the named unknowns drive the next research pass.
|
||||
// Bound: after 2 prior loop-backs on this issue, escalate to the owner
|
||||
// instead of cycling silently — infinite MORE-RESEARCH is dithering.
|
||||
if (moreResearch) {
|
||||
let priorLoopBacks = 0;
|
||||
try {
|
||||
const comments = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number, per_page: 100,
|
||||
});
|
||||
priorLoopBacks = comments.data.filter(c => (c.body || '').includes('honest loop-back accepted')).length;
|
||||
} catch (e) { core.info('loop-back count fetch failed: ' + (e && e.message)); }
|
||||
if (priorLoopBacks >= 2) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
body: `⏸️ **Loop-back bound reached (${priorLoopBacks} prior MORE-RESEARCH cycles).** The synthesis again declares MORE-RESEARCH with substantiated open questions. Escalating to the owner rather than cycling silently: this issue needs a human decision — narrow the scope, accept a design under uncertainty, or split the unknowns into their own research issues.`,
|
||||
});
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number, labels: ['blocked'],
|
||||
}).catch(e => { if (e.status !== 404) throw e; });
|
||||
core.info(`research-clear: loop-back bound reached on #${issue.number} — escalated with blocked label.`);
|
||||
return;
|
||||
}
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
body: `🔁 **Synthesis not ready — honest loop-back accepted (cycle ${priorLoopBacks + 1}).** Evidence floors met (INWARD citations: ${inwardCites.size}, OUTWARD sources: ${outwardUrls.size}, trail annotated: ${annotated.length}, live-verified: ${outwardUrls.size - dead.length}), but the synthesis itself declares MORE-RESEARCH with substantiated open questions.\n\nresearch-pending stays. Next round must resolve the named unknowns — a later synthesis must either produce a plans/ artifact or identify NEW unknowns. Bound: 2 loop-backs, then escalation.`,
|
||||
});
|
||||
core.info(`research-clear: honest MORE-RESEARCH loop-back on #${issue.number} (cycle ${priorLoopBacks + 1}) — research-pending retained.`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Clear research-pending, add in-progress
|
||||
try {
|
||||
await github.rest.issues.removeLabel({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number, name: 'research-pending',
|
||||
});
|
||||
} catch (e) { if (e.status !== 404) throw e; }
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number, labels: ['in-progress'],
|
||||
}).catch(e => { if (e.status !== 404) throw e; });
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
body: `🔬 **Research round verified — synthesis qualified.** 4 phases + evidence floor met (INWARD citations: ${inwardCites.size}, OUTWARD sources: ${outwardUrls.size}, trail annotated: ${annotated.length}, live-fetch verified: ${outwardUrls.size - dead.length}); synthesis declared evidence ${hasSufficiency ? 'sufficiency statement ✓' : '✓'} + feasibility judgement ✓ and committed to a plans/ artifact.\n\n▶️ **in-progress applied** — Rune is now working on this issue. The pipeline will notify at each transition.`,
|
||||
});
|
||||
core.info(`research-clear: #${issue.number} research verified with evidence, label cleared.`);
|
||||
|
||||
# INCOMPLETE CATCHER: detect branches with commits but no PR, belonging to
|
||||
# in-progress issues — auto-create draft PR with incomplete label + ledger.
|
||||
# Runs on the hourly sweep (piggybacks on existing schedule trigger).
|
||||
incomplete-catcher:
|
||||
runs-on: rune-ci-desktop
|
||||
steps:
|
||||
- name: Detect abandoned in-progress work and create draft PRs
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const owner = context.repo.owner;
|
||||
const repo = context.repo.repo;
|
||||
// Find open issues with in-progress label
|
||||
const issues = [];
|
||||
for (let page = 1; ; page++) {
|
||||
const res = await github.rest.issues.listForRepo({
|
||||
owner, repo, state: 'open', labels: 'in-progress', per_page: 50, page,
|
||||
});
|
||||
issues.push(...res.data.filter(i => !i.pull_request));
|
||||
if (res.data.length < 50) break;
|
||||
}
|
||||
if (issues.length === 0) { core.info('No in-progress issues.'); return; }
|
||||
// Find open branches
|
||||
const branches = await github.rest.repos.listBranches({ owner, repo, per_page: 100 });
|
||||
const branchNames = branches.data.map(b => b.name);
|
||||
// Find existing PRs
|
||||
const prs = await github.rest.pulls.list({ owner, repo, state: 'open', per_page: 100 });
|
||||
const prHeads = new Set(prs.data.map(p => p.head.ref));
|
||||
for (const issue of issues) {
|
||||
// Look for a branch matching the issue pattern
|
||||
const branch = branchNames.find(b => b.includes(String(issue.number)) || b === issue.head_branch);
|
||||
if (!branch || prHeads.has(branch)) continue;
|
||||
// Branch exists, no PR yet — create draft PR with incomplete label
|
||||
try {
|
||||
const pr = await github.rest.pulls.create({
|
||||
owner, repo,
|
||||
title: `[INCOMPLETE] ${issue.title}`,
|
||||
head: branch, base: 'master', draft: true,
|
||||
body: [
|
||||
'## Incomplete Work Ledger',
|
||||
'',
|
||||
`**Issue:** #${issue.number}`,
|
||||
`**Branch:** ${branch}`,
|
||||
`**Status:** Work in progress, paused mid-implementation.`,
|
||||
'',
|
||||
'### What exists',
|
||||
'- See commits on this branch',
|
||||
'',
|
||||
'### What remains',
|
||||
'- Resume implementation from where it left off',
|
||||
'- Run gates when complete',
|
||||
'',
|
||||
'### Resume instructions',
|
||||
'1. Check out this branch',
|
||||
'2. Read the plan file in the worktree',
|
||||
'3. Continue from the last commit',
|
||||
'4. Mark ready when implementation is complete',
|
||||
].join('\n'),
|
||||
});
|
||||
await github.rest.issues.addLabels({
|
||||
owner, repo, issue_number: pr.data.number, labels: ['incomplete'],
|
||||
});
|
||||
await github.rest.issues.createComment({
|
||||
owner, repo, issue_number: issue.number,
|
||||
body: `📋 **Incomplete work captured** — draft PR #${pr.data.number} created from branch \`${branch}\`. Resume from there when ready.`,
|
||||
});
|
||||
core.info(`Created draft PR #${pr.data.number} for issue #${issue.number}`);
|
||||
} catch (e) {
|
||||
if (e.status !== 422) throw e; // 422 = PR already exists
|
||||
}
|
||||
}
|
||||
@@ -103,10 +103,10 @@
|
||||
"evidence": "REMOVED by P0-04 (#20): _run_warm_router is a documented no-op; the ContextBroker (#19) owns all model-facing context. See docs/pipeline-removal.md."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/attention_capsule.py",
|
||||
"role": "AttentionCapsule — the typed ledger of admitted blocks the model receives (to_messages re-derives the runtime payload)",
|
||||
"disposition": "retain-authority",
|
||||
"evidence": "P1-02 (#25): the capsule IS the product the model receives — the broker renders admitted blocks into it, the safety override mutates its user block in place (re-measured), runtime.run consumes capsule.to_messages(), and the trace v4 capsule_messages/capsule_content_hashes prove the logged capsule EXACTLY equals the sent payload. Legacy create_capsule/to_prompt (string-trim fake budgeting) deprecated with no live callers."
|
||||
"path": "src/rune/attention_capsule.py:96-163",
|
||||
"role": "Bounded working-set candidate",
|
||||
"disposition": "dormant-until-dependency",
|
||||
"evidence": "Creates a bounded capsule but does not own prompt construction."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/inference_trace.py",
|
||||
@@ -151,34 +151,28 @@
|
||||
},
|
||||
{
|
||||
"id": "capability_selection",
|
||||
"current_authority": "CapabilityBroker.select_for_job owns tool-schema exposure; executor keeps execution authority",
|
||||
"current_authority": "All tool schemas in LocalToolRuntime; broker is advisory",
|
||||
"v1_owner": "ContextBroker.capability_selector",
|
||||
"dependency_issue": "#24",
|
||||
"migration_rule": "Select a bounded tool schema set per CognitiveJob and retain one discovery escape hatch.",
|
||||
"current_paths": [
|
||||
{
|
||||
"path": "src/rune/capability_broker.py",
|
||||
"role": "Authoritative capability selector (select_for_job + verified floor + find_capability escape hatch)",
|
||||
"disposition": "retain-authority",
|
||||
"evidence": "P1-01 (#24): select_for_job is the single selection authority — its ordered decision feeds LocalToolRuntime(tools=) at both construction sites and the _contributor_tools prompt block; degrade-not-crash falls back to full TOOLS with a tier-1 record_failure."
|
||||
"path": "src/rune/capability_broker.py:293-394",
|
||||
"role": "Capability-card generation and relevance matching candidate",
|
||||
"disposition": "dormant-until-dependency",
|
||||
"evidence": "Matches cards but is not passed into send_task."
|
||||
},
|
||||
{
|
||||
"path": "memory/index/capability_cards.json",
|
||||
"role": "Capability-card index (rebuilt on empty/stale/corrupt; native cards synthesized from TOOLS metadata)",
|
||||
"disposition": "migrate-adapter",
|
||||
"evidence": "P1-01 (#24): load_cards validates and rebuilds — the packaged empty array no longer passes as an index; cards carry risk/preconditions/side_effects copied from tool_executor.TOOLS."
|
||||
"role": "Packaged capability-card index",
|
||||
"disposition": "archive-candidate",
|
||||
"evidence": "Packaged file is an empty array and empty content does not trigger rebuild."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/local_tool_runtime.py",
|
||||
"role": "Runtime consumes the selected set; find_capability escape hatch exposes discovered tools in-run",
|
||||
"path": "src/rune/local_tool_runtime.py:129-152",
|
||||
"role": "Current effective tool-schema selection",
|
||||
"disposition": "migrate-adapter",
|
||||
"evidence": "P1-01 (#24): tools= receives the broker-selected schemas (zero-token guarantee for unselected capabilities); unknown-tool errors name find_capability; resume round-trips rebuild the persisted exposed_tools set."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/rune_operator.py",
|
||||
"role": "Operator wiring: selection flows to both LocalToolRuntime sites and the tools prompt block",
|
||||
"disposition": "migrate-adapter",
|
||||
"evidence": "P1-01 (#24): _capability_selection consults the broker once per task; trace schema v3 records selected_tools / capability_reason / find_capability_calls."
|
||||
"evidence": "Defaults the runtime tool list to every tool definition."
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -190,34 +184,28 @@
|
||||
"migration_rule": "One router selects the cheapest verified-capable model from measured profiles and records the reason.",
|
||||
"current_paths": [
|
||||
{
|
||||
"path": "src/rune/model_routing.py",
|
||||
"role": "Authoritative advisory ModelRouter + measured registry (ranked candidates, recorded reasons)",
|
||||
"path": "src/rune/rune_tui.py:3398-3414",
|
||||
"role": "Actual TUI model name resolution",
|
||||
"disposition": "migrate-adapter",
|
||||
"evidence": "P1-06 (#29): ModelRouter.rank() selects the cheapest verified-capable model from measured profiles (state/model_routing_profiles.json) and records the reason; registered as the single model_routing authority in composition.py."
|
||||
"evidence": "Resolves auto/local/cloud immediately before send_task."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/rune_tui.py:_get_model_name",
|
||||
"role": "TUI model-name resolution (delegates to _resolve_model_name)",
|
||||
"disposition": "migrate-adapter",
|
||||
"evidence": "P1-06 (#29): delegates to the module-level resolver; the cloud=qwen2.5:7b fake was replaced by an explicit fallback reason string."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/rune_operator.py:_fallback_model",
|
||||
"role": "ARCHIVED keyword model heuristic (was _decide_model)",
|
||||
"path": "src/rune/rune_operator.py:395-412",
|
||||
"role": "Fallback keyword model selection",
|
||||
"disposition": "archive-candidate",
|
||||
"evidence": "P1-06 (#29): the router replaces the keyword heuristic; the keyword fallback survives only on the router-unavailable degrade path."
|
||||
"evidence": "Maps task words directly to model IDs."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/model_registry.py:50-156",
|
||||
"role": "Tier/capability profile candidate",
|
||||
"disposition": "dormant-until-dependency",
|
||||
"evidence": "Registry is not consulted by ordinary RuneOperator turns; P1-06 encodes its ladder as ModelRouter.CAPABILITY_FLOOR."
|
||||
"evidence": "Registry is not consulted by ordinary RuneOperator turns."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/elastic_cognition.py:632-709",
|
||||
"role": "Parallel progressive-escalation candidate",
|
||||
"disposition": "archive-candidate",
|
||||
"evidence": "process_message is not called by the ordinary TUI path; P1-06 explicitly does not integrate the archive escalation engines."
|
||||
"evidence": "process_message is not called by the ordinary TUI path."
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -228,17 +216,11 @@
|
||||
"dependency_issue": "#29",
|
||||
"migration_rule": "Standardize provider requests/events behind one interface without moving Rune state into providers.",
|
||||
"current_paths": [
|
||||
{
|
||||
"path": "src/rune/model_routing.py:ProviderAdapter",
|
||||
"role": "Chat-only provider boundary (runtime-checkable Protocol, forge idiom)",
|
||||
"disposition": "migrate-adapter",
|
||||
"evidence": "P1-06 (#29): OllamaAdapter delegates to OllamaChatTransport; cloud adapters are declared (CloudStubAdapter raises ProviderUnavailable) until credentials/config exist."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/local_tool_runtime.py:44-126",
|
||||
"role": "Live Ollama chat/capability transport",
|
||||
"disposition": "migrate-adapter",
|
||||
"evidence": "P1-06 (#29): the operator wires router-selected transports through ProviderAdapter; capability discovery stays in local_tool_runtime (#28 boundary)."
|
||||
"evidence": "RuneOperator instantiates OllamaChatTransport directly."
|
||||
},
|
||||
{
|
||||
"path": "src/rune/provider_registry.py:128-320",
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
# Capability Selection — the broker is the authority (#24, P1-01)
|
||||
|
||||
## What changed
|
||||
|
||||
`capability_broker.py` is now the **single capability-selection authority**
|
||||
(authority map `capability_selection` → v1_owner
|
||||
`ContextBroker.capability_selector`). Before #24 every run exposed all 19
|
||||
tool schemas to the model and the broker was advisory. Now:
|
||||
|
||||
- `select_for_job(task_description, need_tools=True)` returns an ordered
|
||||
tool-schema selection with a recorded reason. The operator feeds that
|
||||
exact list into `LocalToolRuntime(tools=...)` at both construction sites,
|
||||
so **unselected capabilities contribute zero prompt tokens** (schemas are
|
||||
excluded from the native `tools` payload, not just hidden from the prompt
|
||||
block).
|
||||
- **Verified floor**: every tool-using job sees at least
|
||||
`FLOOR_TOOLS = read_file, write_file, edit_file, list_directory,
|
||||
grep_files` plus the escape hatch — selection is never empty and never
|
||||
misses the floor for coding tasks. A zero-score task falls back to the
|
||||
floor with reason `"floor (no matches)"`.
|
||||
- **Escape hatch**: `find_capability` is a native read-only tool that is
|
||||
always exposed. Executing it queries the broker (`find_capability(query)`)
|
||||
and returns matching capability summaries with argument schemas; the
|
||||
runtime then **dynamically exposes the discovered native tools for the
|
||||
rest of the run** (within the existing 8-iteration budget — no extra
|
||||
iterations granted). Unknown-tool errors name the hatch so the model can
|
||||
recover in one step.
|
||||
- **Selection is deterministic** (keyword scorer over TOOLS-derived cards,
|
||||
ties broken by name) and bounded (`MAX_SELECTED = 10`).
|
||||
- **Task-family input** reuses `infer_task_family` (#29) for coherence with
|
||||
model routing.
|
||||
|
||||
## Ownership boundaries
|
||||
|
||||
- Selection authority: `capability_broker.select_for_job`.
|
||||
- Execution authority: unchanged in `tool_executor` (#21) — the approval
|
||||
split (`tool_requires_approval` / `PreparedTool`) stays authoritative for
|
||||
gating; per-tool `risk` metadata informs selection, never bypasses
|
||||
approval.
|
||||
- Degrade-not-crash (P0-06): any broker failure falls back to the full
|
||||
TOOLS set (the pre-#24 behavior) and records a tier-1
|
||||
`record_failure("capability_selection", ...)`.
|
||||
|
||||
## Cards
|
||||
|
||||
`load_cards` validates the card index and **rebuilds when the file is
|
||||
missing, empty, corrupt, or stale** (mtime older than the registry).
|
||||
Native-tool cards are synthesized from `tool_executor.TOOLS` metadata —
|
||||
the single source for `risk` (`read-only` | `write` | `irreversible`),
|
||||
`preconditions`, and `side_effects` — so selection has full coverage even
|
||||
with no plugins installed.
|
||||
|
||||
## Resume round-trip
|
||||
|
||||
`tool_names` is validated on resume, so the operator persists
|
||||
`exposed_tools` (the selected set plus any tool dynamically exposed via
|
||||
`find_capability` mid-run) into `pending_state`; `_runtime_for_model`
|
||||
rebuilds exactly that set for the resumed runtime.
|
||||
|
||||
## Trace
|
||||
|
||||
Schema v3 (additive): `selected_tools`, `capability_reason`,
|
||||
`find_capability_calls`. v1/v2 rows and readers stay compatible.
|
||||
|
||||
## Verification
|
||||
|
||||
- Frozen tests: `tests/test_p1_01_capability_authority.py`.
|
||||
- Canary: `src/rune/canary_p1_06.py`-style self-bootstrap in
|
||||
`src/rune/canary_p1_01.py` (registered in `.gitea/workflows/ci.yml`) —
|
||||
asserts ~5–10 exposed schemas, a byte-size drop in the tools payload,
|
||||
the escape-hatch round-trip, and the floor guarantee.
|
||||
- Composition: `compose_capability_selection(root, state_dir)` registers
|
||||
exactly one `CapabilityBroker` per composition root.
|
||||
@@ -1,67 +0,0 @@
|
||||
# Capsule Fidelity (P1-02, #25)
|
||||
|
||||
The AttentionCapsule is the **product the model receives** — not a
|
||||
side-report. This document states the invariant, the mechanism, and how to
|
||||
verify it.
|
||||
|
||||
## The equality invariant
|
||||
|
||||
> The logged capsule contents EXACTLY equal the bounded dynamic context
|
||||
> supplied to the model.
|
||||
|
||||
Concretely, three renderings of one turn must be identical:
|
||||
|
||||
1. `capsule.to_messages()` — re-derived from the admitted-block ledger,
|
||||
2. the message list passed to `LocalToolRuntime.run(...)` — captured
|
||||
pre-run via `capsule.capture_messages()` → `capsule.sent_messages`,
|
||||
3. `InferenceTrace.capsule_messages` — the durable JSONL log.
|
||||
|
||||
Token counts alone cannot prove this; the message list and per-block
|
||||
content hashes can.
|
||||
|
||||
## Mechanism
|
||||
|
||||
- **Assembly:** `ContextBroker.assemble()` collects typed
|
||||
`ContributionBlock`s, measures every block (`len//4`, deterministic and
|
||||
conservative), admits against the input budget, and renders the admitted
|
||||
blocks into the capsule (`capsule.blocks`).
|
||||
- **Payload derivation:** `AttentionCapsule.to_messages()` re-derives the
|
||||
chat message list from `capsule.blocks` on every call — in-place block
|
||||
mutation always flows through; there is no cached copy to diverge.
|
||||
- **Capture:** `RuneOperator._execute_on_model` calls
|
||||
`capsule.capture_messages(capsule.to_messages())` immediately before
|
||||
`runtime.run`. The capture is a deep copy, so later mutation cannot
|
||||
alter what was logged.
|
||||
- **Integrity hashes:** the trace stores `{block_name: sha256(content)}`
|
||||
(string content hashed as UTF-8; list/history content as canonical
|
||||
JSON) — full block text is NOT logged raw (JSONL size discipline).
|
||||
- **Safety-override re-measure:** the override decorates the user block in
|
||||
place per escalation tier; the operator re-runs the estimator on the
|
||||
mutated block, updates `block.tokens`, the broker trace's per-block
|
||||
entry, and `capsule.token_count`. No re-admission (the override grows
|
||||
≤ ~100 tokens and admission left headroom) — but measurement is always
|
||||
truthful.
|
||||
- **num_ctx propagation:** the runtime receives
|
||||
`num_ctx = capsule.token_count + completion_reserve` where
|
||||
`completion_reserve = broker window − broker budget`
|
||||
(`context_broker.default_budget` reserves the completion room). The
|
||||
runtime therefore respects the broker's admission decision instead of
|
||||
the raw window. #28 owns the broader model-aware budget work.
|
||||
|
||||
## Verification
|
||||
|
||||
- Frozen tests: `tests/test_p1_02_capsule_authority.py` — asserts
|
||||
`trace.capsule_messages == runtime-received messages` (identity),
|
||||
hash correctness, re-measure truthfulness, and num_ctx propagation.
|
||||
- Canary: `python3 src/rune/canary_p1_02.py` prints
|
||||
`P1-02 CAPSULE FIDELITY CANARY PASSED` (registered in
|
||||
`.gitea/workflows/ci.yml`'s canary loop).
|
||||
|
||||
## Retired legacy path
|
||||
|
||||
`attention_capsule.create_capsule` and `AttentionCapsule.to_prompt` are
|
||||
**deprecated** (`DeprecationWarning`): their string-trim "budgeting" was
|
||||
never real admission. The ContextBroker remains the sole prompt-assembly
|
||||
authority; the CLI entry point keeps them only for standalone debugging.
|
||||
`ContributionBlock.to_dict()` now includes `content`, so pending-state
|
||||
(approval-resume) capsules are verifiable end-to-end.
|
||||
@@ -74,45 +74,6 @@ degraded, it is in the ledger even when the turn looked normal.
|
||||
4. Check `/health` after forcing a failure; the subsystem must appear in the
|
||||
ledger and the event must land in the journal.
|
||||
|
||||
## Failure auto-filing (#122)
|
||||
|
||||
T0/T1 failures additionally become ONE forge issue per failure signature
|
||||
`(subsystem, error_class)` via `src/rune/failure_auto_filer.py` — the
|
||||
forge's own watch system pings you; repeats comment with a rising count
|
||||
instead of flooding the board. T2 failures (cron noise) NEVER file.
|
||||
|
||||
How it works:
|
||||
|
||||
- **Signature** — `(subsystem, error_class)` where `error_class` is the
|
||||
exception class name (or first token of the error string). Never the
|
||||
full message: variable payloads would flood.
|
||||
- **Dedup** — before filing, a repo-scoped marker search
|
||||
(`ForgeClient.search_issues`, `GET /issues?state=open&type=issues&q=<marker>`
|
||||
— Gitea/Forgejo has no repo-scoped `/issues/search` path) looks for
|
||||
the marker `[auto-failure:<subsystem>:<error_class>]` in titles and
|
||||
bodies. Found → `add_comment` with a rising count; not found →
|
||||
`create_issue`. The list endpoint is paginated, so older failure
|
||||
issues are still seen.
|
||||
- **Burst coalescing** — an in-memory per-signature counter batches
|
||||
repeats into one comment per 10 occurrences or 10 minutes (whichever
|
||||
first). A 3am crash-loop writes one comment, not 200.
|
||||
- **Spool + reconcile** — any client exception (forge unreachable/auth/
|
||||
5xx) appends a `failure_auto_file_spool` JSON line to the existing
|
||||
event journal. `reconcile()` (startup + recovery) replays them; dedup
|
||||
re-applies, filed lines are marked consumed.
|
||||
- **Threading** — the failure path only enqueues; one daemon worker
|
||||
drains the queue and does all network work. Failure-path latency is
|
||||
untouched, and nothing ever raises into the caller's except block.
|
||||
- **Tier gate** — `tier in (0, 1)` file/comment; `tier >= 2` journal
|
||||
only. Enforced in ONE place: the filer.
|
||||
|
||||
Wiring: the composition root (`rune_operator.get_operator`) creates one
|
||||
`FailureAutoFiler` via `create_forge_client()` (unconfigured →
|
||||
`NullAdapter` → dormant by construction) and wires it through
|
||||
`subsystem_health.set_auto_filer`; `record_failure(..., filer=...)`
|
||||
accepts an explicit override. `RUNE_AUTOFILE_OFF=1` disables entirely
|
||||
(fail-safe off switch).
|
||||
|
||||
## Scope note
|
||||
|
||||
P0-06 converted the ~30 critical handlers on the authoritative path (surveyed
|
||||
|
||||
@@ -10,20 +10,13 @@ prerequisite for the failure auto-filer (#122).
|
||||
|
||||
- `ForgeClient` — `@runtime_checkable` Protocol: `create_issue(title, body)`,
|
||||
`add_comment(issue_number, body)`, `get_issue_state(issue_number)`,
|
||||
`list_open_issues()`, `search_issues(query)` (#122)
|
||||
`list_open_issues()`
|
||||
- `NullAdapter` — every method is a no-op dict; zero errors; the graceful
|
||||
dormant path for unconfigured/standalone use
|
||||
- `GiteaForgejoAdapter` — real HTTP adapter via Gitea/Forgejo `/api/v1`
|
||||
paths (identical on both platforms); `Authorization: token <token>`
|
||||
- `create_forge_client(config)` — factory; unconfigured → `NullAdapter`
|
||||
|
||||
`search_issues(query)` (#122) is the repo-scoped marker search over the
|
||||
paginated LIST endpoint (`GET /issues?state=open&type=issues&q=<query>`
|
||||
+ client-side marker contains-check on title/body). Gitea/Forgejo has
|
||||
NO repo-scoped `/issues/search` path — the failure auto-filer dedups
|
||||
through the list endpoint precisely because it is paginated and
|
||||
server-filterable.
|
||||
|
||||
## Configuration
|
||||
|
||||
The factory checks a config dict first, then environment variables:
|
||||
@@ -50,7 +43,6 @@ If URL **or** token is missing → `NullAdapter` (never raises).
|
||||
POST {url}/api/v1/repos/{owner}/{repo}/issues/{n}/comments
|
||||
GET {url}/api/v1/repos/{owner}/{repo}/issues/{n}
|
||||
GET {url}/api/v1/repos/{owner}/{repo}/issues?state=open&limit=50
|
||||
GET {url}/api/v1/repos/{owner}/{repo}/issues?state=open&type=issues&q=<query>&limit=50&page=N
|
||||
|
||||
## Run
|
||||
|
||||
|
||||
+2
-13
@@ -17,11 +17,11 @@ is gitignored, P0-05 M6) — never committed. Consumers: #25, #28, #29.
|
||||
Read back with `rune.inference_trace.read_traces()` or
|
||||
`python3 -m rune.inference_trace [n]` (tail n records).
|
||||
|
||||
## Record schema (version 4)
|
||||
## Record schema (version 1)
|
||||
|
||||
| Field | Source | Meaning |
|
||||
|---|---|---|
|
||||
| `schema_version` | constant (`4`) | Record shape version for consumers |
|
||||
| `schema_version` | constant (`1`) | Record shape version for consumers |
|
||||
| `provider` | `OllamaChatTransport.provider` | `"ollama"` (class constant) |
|
||||
| `model` | operator | Model id the run executed on |
|
||||
| `task_id` / `channel` | operator (TaskState #18) | Joins the trace to TaskState |
|
||||
@@ -43,17 +43,6 @@ Read back with `rune.inference_trace.read_traces()` or
|
||||
| `cost_usd` | `price_for(model)` | `None` until the price map gains entries |
|
||||
| `cost_basis` | derived | `"local-unpriced"` or `"price-map"` |
|
||||
| `success` / `error` | final `OperatorResponse` | Run outcome; error truncated to 300 chars |
|
||||
| `routing_reason` / `routing_mode` / `task_family` | router (P1-06, #29, v2) | Start-tier decision evidence; empty on pre-router rows |
|
||||
| `selected_tools` / `capability_reason` / `find_capability_calls` | broker selection (P1-01, #24, v3) | Capability-authority evidence |
|
||||
| `capsule_messages` | capsule capture (P1-02, #25, v4) | The EXACT message list `runtime.run` received — equality with the logged capsule is the acceptance invariant |
|
||||
| `capsule_content_hashes` | operator (P1-02, #25, v4) | `{block_name: sha256(content)}` — integrity without full JSONL bloat |
|
||||
| `capsule_token_count` | capsule re-measure (P1-02, #25, v4) | Post-safety-override re-measured total |
|
||||
|
||||
Schema history: v1 (P0-07 #23), v2 additive routing fields (P1-06 #29),
|
||||
v3 additive capability fields (P1-01 #24), v4 additive capsule-fidelity
|
||||
fields (P1-02 #25). All bumps are additive with defaults — older rows
|
||||
without the new keys still parse (`read_traces` consumers must tolerate
|
||||
missing keys).
|
||||
|
||||
## Authority and single-source rule
|
||||
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
# Model/Provider Routing (P1-06, issue #29)
|
||||
|
||||
One authoritative `ModelRouter` + `ProviderAdapter` interface. Deterministic →
|
||||
tiny local → strong local → cloud escalation is driven by verification
|
||||
evidence, with the routing reason and estimated cost recorded on every run's
|
||||
durable trace.
|
||||
|
||||
Authority-map migration rule (`model_routing`): *"One router selects the
|
||||
cheapest verified-capable model from measured profiles and records the
|
||||
reason."*
|
||||
|
||||
## The capability ladder
|
||||
|
||||
`ModelRouter.CAPABILITY_FLOOR` encodes the static tier ladder (from
|
||||
`model_registry.ModelProfile`):
|
||||
|
||||
| Tier | Model | Provider | Tools | Context | Cost |
|
||||
|------|-----------------|----------|-------|---------|-------------|
|
||||
| T0 | deterministic | local | no | — | 0 |
|
||||
| T1 | qwen2.5:3b | ollama | no | 32k | 0 |
|
||||
| T2 | qwen3:14b | ollama | no | 32k | 0 |
|
||||
| T3 | qwen3.6:27b | ollama | no | 32k | 0 |
|
||||
| T4 | mimo-v2.5-pro | openai | yes | 128k | paid (declared, not live) |
|
||||
|
||||
## Measured registry
|
||||
|
||||
`MeasuredModelProfile` rows (per `(provider, model)`) aggregate per-task-family
|
||||
success from `inference_traces.jsonl` and persist to
|
||||
`<rune_home>/state/model_routing_profiles.json` (append-safe JSON, same
|
||||
durability pattern as model profiles). `update_profiles_from_traces()` folds
|
||||
trace rows in; `load_profiles()`/`save_profiles()` own the I/O so `rank()`
|
||||
stays pure.
|
||||
|
||||
**Provider removal never erases state**: disconnecting a provider removes only
|
||||
the adapter instance. Profiles and outcomes live on disk keyed by
|
||||
`(provider, model)`, independent of any adapter object.
|
||||
|
||||
## Router semantics
|
||||
|
||||
`ModelRouter.rank(task_family, need_tools, min_context) -> list[RoutingDecision]`
|
||||
is advisory — it never mutates inputs and never executes model calls
|
||||
(`orchestration/router.py` idiom). The **operator tier loop is the final
|
||||
decider**; the router's first candidate is the run's start-tier decision.
|
||||
|
||||
Reason strings name the evidence:
|
||||
|
||||
- `no-measured-profiles tier<N>` — pure capability-floor ordering (no
|
||||
measured profiles exist at all);
|
||||
- `measured <ok>/<n> success task_family=<family>` — verified-capable
|
||||
candidates with measured stats;
|
||||
- a candidate with measured stats but zero successes in the family is **not
|
||||
verified-capable** and is excluded;
|
||||
- when profiles exist, unmeasured candidates are not ranked (no evidence).
|
||||
|
||||
Escalation across runs is a **start-tier decision**: when a run fails
|
||||
(`success=False` or tool failure) on a local model, the next run for that task
|
||||
family may rank at a higher tier. The router never burns cloud tokens mid-run.
|
||||
|
||||
## Trace visibility (schema v2)
|
||||
|
||||
`InferenceTrace` is schema v2 with additive optional fields — v1 rows and v1
|
||||
readers keep working unchanged:
|
||||
|
||||
- `routing_reason` — the start-tier decision reason (e.g.
|
||||
`router: measured 12/13 success task_family=code`);
|
||||
- `routing_mode` — `local` | `cloud`;
|
||||
- `task_family` — the measured-registry family key.
|
||||
|
||||
`RoutingDecision.estimated_cost` lands with the reason, so routing cost is
|
||||
always visible alongside the run's `cost_usd`/`cost_basis`.
|
||||
|
||||
## Provider adaptation
|
||||
|
||||
`ProviderAdapter` is a chat-only, runtime-checkable Protocol (forge idiom,
|
||||
`forge_client.py`): `name`, `available()`, `chat(model, messages, tools,
|
||||
options, think)`. `OllamaAdapter` wraps `OllamaChatTransport` (availability
|
||||
pings `/api/tags`, 2s timeout, 30s cache). Cloud adapters are **declared, not
|
||||
live**: `CloudStubAdapter.chat` raises `ProviderUnavailable` — no credentials
|
||||
enter this issue, and no provider ever silently fakes a response.
|
||||
|
||||
Composition registers exactly one routing authority per root:
|
||||
`composition.compose_model_routing(root, state_dir)` → `root.get("model_routing")`.
|
||||
|
||||
## Out of scope (unchanged)
|
||||
|
||||
- Safety-override in-run tier semantics: untouched.
|
||||
- `num_ctx`/admission: ContextBroker + #28.
|
||||
- Tool execution authority: `LocalToolRuntime` (#32); the
|
||||
`provider_registry._execute_tool_calls` parallel path stays dead.
|
||||
- Archive escalation engines (`elastic_cognition`, `tiered_executor`,
|
||||
`progressive_escalation`): not integrated.
|
||||
@@ -1,84 +0,0 @@
|
||||
# Umbrella system → henxels contracts + retroactive governance — design for confirmation
|
||||
|
||||
## Part 1: Retroactive application (ready to execute now)
|
||||
|
||||
The survey found six ungoverned surfaces. Proposed henxels rules, in the tool's
|
||||
real schema, each following an existing pattern:
|
||||
|
||||
| Surface | Finding | Proposed henxel (keyed, real schema) |
|
||||
|---|---|---|
|
||||
| `shared/*.py` (24 files) | ~100% near-duplicates of `src/rune/` modules (the mtime-copy sync is the cause) | `changed_with`: when `shared/*.py` changes, `src/rune/` mirror must change too — **or better (Part 2) the dupes die** |
|
||||
| `research/**` (63 files, 16 dirs) | No index; unlinked pages | `referenced_in: docs/research-index.md` + `links_resolve` — orphan research is tracked |
|
||||
| `memory/` (runtime state tracked in git!) | `journal/events.jsonl` + runtime JSON in the repo — runtime data in version control | `forbidden_files: [./memory/journal/events.jsonl, ./memory/index/*.json]` after a one-time extraction (runtime state belongs to the deployed agent, not the codebase) |
|
||||
| `backups/*.bak` (3 files) | Build backups committed | `forbidden_files: [./backups/**]` (git history IS the backup) |
|
||||
| `specs/`, `reports/` | Type-ungoverned | `allowed_filetypes: [.md]` (matches the plans rule) |
|
||||
| `survey/` (7 files) | Ungoverned | fold into research indexing or `allowed_filetypes: [.md, .yaml]` |
|
||||
|
||||
All are mechanical; the `memory/` extraction is the only one touching runtime —
|
||||
I'd do it in the same PR with the agent's own journal-preservation discipline
|
||||
(backup branch first).
|
||||
|
||||
## Part 2: Umbrella system → contracts (needs your confirmation on shape)
|
||||
|
||||
### What the umbrella system is (verified)
|
||||
|
||||
`shared/umbrella_update.py`: a hand-maintained manifest (`UMBRELLAS` dict) of six
|
||||
functional umbrellas — **memory, governance, identity, growth, perception,
|
||||
infrastructure** — each listing shared/scripts/canaries components plus a
|
||||
`depends_on` DAG (memory→governance→identity; growth→memory+governance). Purpose:
|
||||
say "update memory" and the system knows the affected files, runs the umbrella's
|
||||
canaries, syncs shared→deploy targets by mtime, and commits.
|
||||
|
||||
### What's wrong (verified defects)
|
||||
|
||||
1. **Manifest drift**: `memory_selector.py` and every post-manifest module is in
|
||||
NO umbrella; it references canaries (`canary_unbounded.py`,
|
||||
`canary_drift_wiring.py`) that don't exist in the repo.
|
||||
2. **Two-sources-of-truth by design**: `sync_shared_to_scripts()` mtime-copies
|
||||
`shared/` → `~/.config/opencode/scripts/` + the old TUI dir. This is why
|
||||
henxels flags ~100% duplicates. The repo and the harness both "own" copies.
|
||||
3. **Hardcoded harness paths** (`~/.config/opencode/...`) — not repo-portable,
|
||||
dead on the runner.
|
||||
|
||||
### Proposed design: three contracts, one removal
|
||||
|
||||
**Contract A — `umbrella_manifest_is_complete`** (new custom statement):
|
||||
every `src/rune/*.py` and `shared/*.py` module must appear in the umbrella
|
||||
manifest (or the coverage-debt ledger). New modules force an umbrella decision —
|
||||
the manifest can never silently drift again. This is the umbrella's real
|
||||
invariant: *the system knows what needs updating* — enforced.
|
||||
|
||||
**Contract B — `umbrella_canaries_exist`** (new custom statement): every canary
|
||||
referenced by the manifest must exist in `src/rune/` and be CI-registered
|
||||
(composes with the existing `ci_runs_all_canaries`). Kills defect 1's phantom
|
||||
references.
|
||||
|
||||
**Contract C — `umbrella_dag_acyclic_and_declared`** (new custom statement):
|
||||
parse `UMBRELLAS`, verify `depends_on` is a DAG (no cycles), every dependency
|
||||
target exists. Cheap, structural, protects the reflex's core logic.
|
||||
|
||||
**The removal (your call):** the mtime-copy sync. Two honest options:
|
||||
|
||||
- **Option 1 (recommended): repo-as-source-of-truth.** The repo's `src/rune/` +
|
||||
`shared/` become the ONLY governed copies. The deployed harness gets its
|
||||
scripts from the pinned checkout (the update flow already force-checks-out
|
||||
gitea/master). `sync_shared_to_scripts()` is retired; the
|
||||
near-duplicate `shared/` files either become true imports of `src/rune` or
|
||||
move to the harness (outside the repo). Contract A then governs ONE tree.
|
||||
Cost: the harness-side scripts dir stops being independently editable —
|
||||
which is the point.
|
||||
- **Option 2: keep dual-location, gate the sync.** Keep the copies but add a
|
||||
`must_be_in_sync`-style contract: shared/ and its mirrors must be
|
||||
content-identical at commit time (henxels `changed_with` + a checksum
|
||||
statement). Cost: permanent duplication machinery; drift only detectable,
|
||||
never prevented.
|
||||
|
||||
### Sequencing
|
||||
|
||||
1. Retroactive henxels (Part 1) — one PR, mechanical.
|
||||
2. Contracts A+B+C + the manifest modernized in-repo (move `UMBRELLAS` to
|
||||
`config/umbrellas.json`, sourced by the script) — second PR.
|
||||
3. Option 1's sync retirement — its own PR, after you confirm.
|
||||
|
||||
**Confirm**: Option 1 or 2 for the sync? And does the `memory/` runtime-state
|
||||
extraction have your blessing (it touches the live journal)?
|
||||
@@ -51,7 +51,7 @@ show.
|
||||
| `task_state` | `RuneState` with competing task ledgers | `TaskStateRepository` | #18 |
|
||||
| `context_construction` | Operator prompt assembly + parallel TUI preprocessing | `ContextBroker` | #19 |
|
||||
| `memory_selection` | Multiple direct recall/association paths | `ContextBroker.memory_selector` | #26 |
|
||||
| `capability_selection` | `CapabilityBroker.select_for_job` owns tool-schema exposure | `ContextBroker.capability_selector` | #24 |
|
||||
| `capability_selection` | All runtime tools; broker advisory only | `ContextBroker.capability_selector` | #24 |
|
||||
| `model_routing` | TUI model selection + operator keyword fallback | `ModelRouter` | #29 |
|
||||
| `provider_adaptation` | Direct Ollama chat transport | `ProviderAdapterRegistry` | #29 |
|
||||
| `tool_schema_execution` | Tool protocol/runtime/executor trio | `ExecutionCore` | #21 |
|
||||
@@ -108,21 +108,12 @@ Evidence: `memory_store.py:43-72`, `rune_operator.py:424-669`,
|
||||
|
||||
### `capability_selection`
|
||||
|
||||
LIVE as of P1-01 (#24): `capability_broker.select_for_job` is the single
|
||||
selection authority — its ordered decision feeds `LocalToolRuntime(tools=)`
|
||||
at both operator construction sites and the tools prompt block. A verified
|
||||
floor (five T1 file tools + `find_capability`) is never dropped; the
|
||||
`find_capability` native tool is the discovery escape hatch and dynamically
|
||||
exposes discovered tools within the running session. The card index is
|
||||
rebuilt on empty/stale/corrupt with native cards synthesized from
|
||||
`tool_executor.TOOLS` metadata. Execution authority stays in
|
||||
`tool_executor` (#21).
|
||||
The broker currently displays matches, but `LocalToolRuntime` defaults to every
|
||||
tool schema and the packaged card index is empty. #24 makes capability selection
|
||||
causal: a bounded set is exposed, with a small discovery escape hatch.
|
||||
|
||||
Evidence: `capability_broker.py` (`select_for_job`, `find_capability`,
|
||||
`load_cards`), `local_tool_runtime.py` (hatch dispatch, unknown-tool
|
||||
guidance), `rune_operator.py` (`_capability_selection`, trace v3),
|
||||
`memory/index/capability_cards.json`,
|
||||
`docs/capability-selection.md`.
|
||||
Evidence: `capability_broker.py:293-394`,
|
||||
`local_tool_runtime.py:129-152`, `memory/index/capability_cards.json`.
|
||||
|
||||
### `model_routing`
|
||||
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
# P1-07 (#30) — Interstitial Contextual Messaging: Work Pane + First-Class Runtime Objects
|
||||
|
||||
> **For agentic workers:** execution contract — FROZEN TESTS ARE THE SPEC. Implement verbatim; record every deviation in the plan tail. TDD red→green. Do not create/complete tasks; controller owns task state.
|
||||
|
||||
**Gitea issue:** #30 (P1 — high). **Branch:** `p1-07-work-pane` off master `0feeed6`.
|
||||
|
||||
## Why
|
||||
|
||||
Tool calls already render as first-class widgets (ToolCallWidget, shipped with #21/#22) — but they live INLINE in the chat log, and the rest of the runtime's cognitive events (memory queries, observations, verification, outcomes) have no typed objects or rendering at all. #30's full contract: a dedicated **work pane** showing observable plan/action/result state while the chat pane stays conversational, plus the full typed-object set.
|
||||
|
||||
## What already exists (build on, don't rebuild)
|
||||
|
||||
- `ToolRequest`/`ToolEvent` typed dataclasses (tool_protocol.py:23/31) — the model for the new objects.
|
||||
- `ToolCallWidget` (message_widgets.py:14) — status glyphs, approval buttons, in-place `apply()` updates, keyed by `(channel, call_id)`.
|
||||
- `ThinkingBubble` — thinking blocks, not prose.
|
||||
- Live wiring: `rune_tui.py:3522-3533` mounts/updates widgets from `result["tool_events"]`.
|
||||
- `CognitiveJob` exists in tiered_executor.py:36 (dormant module — do NOT import from there; define the live object in tool_protocol.py per the frozen spec).
|
||||
- `VerificationResult` exists in verification_layer.py:24 (dormant — same rule).
|
||||
|
||||
## Constraints
|
||||
|
||||
- **Two-pane layout**: TASK channel gains a sibling WORK pane (collapsible); CHAT stays conversational. The work pane shows the ACTIVE channel's runtime objects — tool calls, memory queries, observations, verification, outcomes — as a live operational view. Chat log keeps the widgets it already mounts (both views of the same events; the work pane is the operational surface, the log remains the narrative record).
|
||||
- **No hidden chain-of-thought**: only observable plan/action/result state is rendered.
|
||||
- **Event identity**: every runtime object carries a stable id; the work pane tracks objects by id, updating in place (same pattern as `_tool_widgets`).
|
||||
- **Additive protocol**: new dataclasses in `tool_protocol.py` follow the `ToolRequest`/`ToolEvent` shape (frozen dataclass + `to_dict`/`from_dict` round-trip).
|
||||
- **Dormant modules stay dormant**: do NOT wire tiered_executor or verification_layer — the new objects are independent live types; the dormant modules may adopt them later.
|
||||
- **Functionality immutable**: chat/task channels, /commands, model selector, footer — untouched.
|
||||
- Memory queries: the ContextBroker (#19) already records per-block contributions in its trace; a `MemoryQuery` object is emitted per broker assemble (name, admitted blocks, tokens, dropped) — wired from the operator's existing `_last_trace`.
|
||||
- Verification/Outcome objects: emitted from the operator's response path (success/failure/degraded) — the OutcomeLedger wiring itself is #33, NOT this stage; here they are runtime objects + renderings only.
|
||||
|
||||
## Task 1 — Typed runtime objects (tool_protocol.py)
|
||||
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class MemoryQuery:
|
||||
"""One context-broker memory-selection pass (observable, not hidden)."""
|
||||
query_id: str
|
||||
task_id: str
|
||||
admitted: dict # {block_name: tokens}
|
||||
dropped: list # [{name, tokens}]
|
||||
memory_tokens: int
|
||||
ts: str # ISO timestamp
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Observation:
|
||||
"""An observable runtime event the operator should see."""
|
||||
observation_id: str
|
||||
task_id: str
|
||||
kind: str # "tool_result" | "approval" | "compaction" | "degraded" | "note"
|
||||
summary: str # one-line, no chain-of-thought
|
||||
detail: str # bounded detail (<=300 chars)
|
||||
ts: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VerificationResult:
|
||||
"""Objective verification of a task outcome (tests/exit/diff — never self-report)."""
|
||||
verification_id: str
|
||||
task_id: str
|
||||
method: str # "exit_code" | "test" | "diff" | "manual" | "none"
|
||||
passed: bool
|
||||
evidence: str # bounded (<=300 chars)
|
||||
ts: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Outcome:
|
||||
"""The final classified outcome of a task (feeds #33 later)."""
|
||||
outcome_id: str
|
||||
task_id: str
|
||||
status: str # "completed" | "failed" | "superseded" | "cancelled"
|
||||
verification_id: str # links to VerificationResult ("" if none)
|
||||
degraded: list
|
||||
ts: str
|
||||
```
|
||||
|
||||
All five: `to_dict()` + `from_dict()` round-trip (mirror ToolRequest's shape; `ts` ISO UTC).
|
||||
|
||||
## Task 2 — Operator emission points
|
||||
|
||||
`rune_operator.py`:
|
||||
1. After broker assemble (inside `_execute_on_model`, where `_last_trace` is set): build a `MemoryQuery` from the trace (admitted/dropped/tokens) and attach to the response envelope: `response.runtime_objects` — a new additive list field on `OperatorResponse` (default empty; carries dicts via `to_dict()`).
|
||||
2. On final response: build `VerificationResult` (method: "exit_code" if the tool loop ran run_command with exit 0, "test" if run_tests passed, "none" otherwise; passed: response.success and not degraded-critical; evidence: bounded summary) and `Outcome` (status from task transition; links verification).
|
||||
3. Observations: emit for compaction events (from trace["compaction"]) and degraded subsystems (one per degraded name).
|
||||
4. `runtime_objects` flows through `_response_envelope` (additive key) → TUI.
|
||||
|
||||
## Task 3 — Work pane (rune_tui.py)
|
||||
|
||||
1. New widget `WorkPane(Vertical)` — docked as a third column in `#channels` (collapsible via Ctrl+3 / a header click; default visible on the task channel, hidden on chat). Header: "WORK — runtime objects". Sections render newest-first: Outcomes, Verification, Observations, Memory queries, Tool calls.
|
||||
2. `WorkObjectWidget(Static)` — generic renderer: kind glyph + id + summary line + expandable detail (bounded). Tool calls keep their existing `ToolCallWidget` in the chat log AND get a one-line mirror in the work pane (the pane is the operational index; the log keeps the interaction).
|
||||
3. Wiring in `_respond`/`_resolve`: after mounting tool widgets, consume `result["runtime_objects"]` → update/append work-pane entries keyed by object id.
|
||||
4. CSS: `.work-pane { width: 1fr; border-left: solid $secondary; }` — three equal columns on the task channel; work pane hidden (`display: none`) when chat is focused.
|
||||
|
||||
## Task 4 — Frozen tests (tests/test_p1_07_runtime_objects.py)
|
||||
|
||||
```python
|
||||
def test_runtime_objects_round_trip():
|
||||
"""All five new types round-trip to_dict/from_dict with stable ids."""
|
||||
|
||||
def test_operator_emits_memory_query_from_broker_trace():
|
||||
"""A scripted send_task produces a MemoryQuery matching the broker trace
|
||||
(admitted blocks + tokens + dropped)."""
|
||||
|
||||
def test_operator_emits_verification_and_outcome():
|
||||
"""A successful send_task with a run_command exit-0 emits VerificationResult
|
||||
(method=exit_code, passed=True) and a linked Outcome (status=completed)."""
|
||||
|
||||
def test_failed_task_emits_failed_outcome():
|
||||
"""A failed task emits Outcome(status=failed) with no verification link."""
|
||||
|
||||
def test_degraded_subsystems_become_observations():
|
||||
"""Each degraded subsystem name becomes one Observation(kind=degraded)."""
|
||||
|
||||
def test_compaction_emits_observation():
|
||||
"""A compaction event in the broker trace becomes Observation(kind=compaction)."""
|
||||
|
||||
def test_response_envelope_carries_runtime_objects():
|
||||
"""runtime_objects survives _response_envelope as dicts (TUI-consumable)."""
|
||||
|
||||
def test_work_pane_renders_runtime_objects():
|
||||
"""WorkPane.add_object() renders kind glyph + id + summary; update_object()
|
||||
replaces in place by id (no duplicates)."""
|
||||
|
||||
def test_work_pane_collapses_and_expands():
|
||||
"""Ctrl+3 toggles the work pane; hidden on chat channel, visible on task."""
|
||||
```
|
||||
|
||||
RED: MemoryQuery/WorkPane missing, envelope lacks runtime_objects.
|
||||
|
||||
## Task 5 — Canary + docs + CI
|
||||
|
||||
1. `src/rune/canary_p1_07.py` (no live model): round-trips all five objects; scripted operator run emits MemoryQuery + VerificationResult + Outcome; WorkPane render/update assertions via a headless construct (no app runtime — test the widget's data methods); prints `P1-07 RUNTIME OBJECTS CANARY PASSED`. Register in ci.yml SAME commit (henxels gate).
|
||||
2. `docs/runtime-objects.md`: the object set, emission points, work-pane contract, two-view principle (pane = operational index, log = narrative), #33 handoff point (Outcome is the Ledger's future record shape).
|
||||
3. Authority map: add runtime-objects entry.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- All 9 frozen tests green (red proven first).
|
||||
- Full suite green (baseline 513 → +9 ≈ 522; record actual; zero regressions).
|
||||
- Work pane visible on task channel: tool calls, memory queries, verification, outcomes render live; Ctrl+3 toggles; chat channel unaffected.
|
||||
- Canary green + registered; henxels exit 0; ruff clean (format ADDED files).
|
||||
- No chain-of-thought anywhere in the pane (only observable state).
|
||||
|
||||
## Pipeline requirements checklist
|
||||
- [ ] TDD red→green (9 frozen tests)
|
||||
- [ ] ruff check + format on touched/added files
|
||||
- [ ] Spec gate PASS (independent reviewer)
|
||||
- [ ] Quality gate Approved (independent reviewer)
|
||||
- [ ] Full-suite regression green
|
||||
- [ ] henxels check --all green
|
||||
- [ ] Closed with formatted comment + refs
|
||||
## Execution deviations (recorded at resume, per plan contract)
|
||||
|
||||
- D1: Branch rebased from base 0feeed6 onto live master faed039 (master gained #168 dual-mode imports, #170/#174 label split). One semantic conflict in rune_operator.py imports: resolved by combining master's dual-mode try/except wrapping with the branch's five new tool_protocol names (MemoryQuery, Observation, Outcome, VerificationResult + OperatorResponse/contains_internal_protocol), preserving both.
|
||||
- D2: Third commit (eac5ee2 dual-mode lazy import) applied as comment-text-only conflict; master already carried identical code — kept master's variant.
|
||||
- D3: Suite result at resume: 523 passed + 84 subtests (plan estimated ≈522; +1 from master-side test additions between 0feeed6 and faed039). Zero regressions; all 9 frozen P1-07 tests green.
|
||||
- D4: henxels reports 21 pre-existing warn-level near-duplicate notices (shared/ vs src/rune/), tracked under #41; exit 0.
|
||||
@@ -1,128 +0,0 @@
|
||||
# P1-01 CapabilityBroker Authoritative — Implementation Plan (#24)
|
||||
|
||||
> **For agentic workers:** execution contract — FROZEN TESTS ARE THE SPEC. Implement verbatim; record every deviation in the plan tail. TDD red → green. Do not create/complete tasks; controller owns task state.
|
||||
|
||||
**Goal:** the model is exposed only the capability schemas that fit the current task (100 installed → coding job sees ~5–10); unused capabilities cost zero prompt tokens; a tiny `find_capability` escape hatch lets the model request an unexposed capability; capability cards gain preconditions/risk/side-effect metadata and stale/empty validation.
|
||||
|
||||
**Architecture:** `capability_broker.py` becomes the single selection authority (per `config/v1_authority_map.json` `capability_selection` → v1_owner `ContextBroker.capability_selector`): `select_for_job(task_description, need_tools)` returns an ordered tool-name selection with reason; the operator feeds that selection into `LocalToolRuntime(tools=...)` (both construction sites) and replaces the names-only `_contributor_tools` prompt block with the selected set; a always-exposed tiny `find_capability` native tool is the escape hatch; unknown-tool requests route to a helpful error listing `find_capability`. Cards gain `preconditions`, `side_effects`, per-tool `risk` and rebuild on empty/stale. Composition registers the broker as the single selection authority.
|
||||
|
||||
**Tech stack:** stdlib + existing house modules only. No new services.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Authority map rule: capability selection is owned by ContextBroker-side selection; do NOT create a parallel authority; executor (`tool_executor`) keeps execution authority (#21).
|
||||
- Native schemas ride the `tools` API param — selection MUST flow to `LocalToolRuntime(tools=)`, prompt block alone is insufficient.
|
||||
- `tool_names` is validated on resume (`local_tool_runtime.py:323`) — resume round-trips must use the same selected set.
|
||||
- Selection must never return an empty set for a coding-style task: verified floor = T1 file tools (read_file, write_file, edit_file, list_dir, search_files) + find_capability.
|
||||
- The 8-iteration budget: `find_capability` results must let the model name a tool that the SAME run can then call (dynamic exposure within the run) — otherwise the hatch is useless.
|
||||
- Degrade-not-crash (P0-06): any broker failure falls back to full TOOLS (today's behavior) + `record_failure("capability_selection", ..., tier=1)`.
|
||||
- TDD; ruff check + format on touched/added files; canary registered in ci.yml same commit.
|
||||
|
||||
## Design Decisions (settled)
|
||||
|
||||
1. **Owner boundary:** CapabilityBroker executes selection; the broker's decision is *the* authority for what the runtime exposes (the authority-map's "ContextBroker.capability_selector" names the selection point inside context assembly — implemented as the broker module invoked from context/tool-exposure assembly; no second selector).
|
||||
2. **Selection basis:** deterministic keyword scorer (existing `score_card`) over per-tool metadata derived from TOOLS + cards; floor set on zero-score. Task-family input via `infer_task_family` (model_routing, #29) for coherence.
|
||||
3. **Escape hatch:** `find_capability` is a native tool always in the selected set. Executing it queries the broker and returns matching capability summaries (name + description + schema). The runtime's unknown-tool error path suggests `find_capability` and dynamically registers a requested tool into the running session's tool set when it exists (within iteration budget — no extra iterations granted).
|
||||
4. **Card validation:** `load_cards` rebuilds when file missing OR empty OR stale (mtime older than `.microkernel.json` registry). Cards for native tools are synthesized from TOOLS metadata so selection has full coverage even with no plugins.
|
||||
5. **Metadata schema:** per-tool `risk` ("read-only"|"write"|"irreversible"), `preconditions` (list), `side_effects` (list) added to TOOLS entries (single source; cards reference tool names). Approval split (`tool_requires_approval`) stays authoritative for gating.
|
||||
6. **Trace additive fields (v3):** `selected_tools: list[str] = []`, `capability_reason: str = ""`, `find_capability_calls: int = 0` — additive defaults, v1/v2 readers unaffected.
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1 — Per-tool metadata + card synthesis/validation
|
||||
|
||||
**Files:** modify `src/rune/tool_executor.py` (metadata on TOOLS), `src/rune/capability_broker.py` (synthesis, rebuild-on-empty/stale), test `tests/test_p1_01_capability_authority.py`.
|
||||
|
||||
- [ ] Step 1: failing tests — every TOOLS entry carries risk/preconditions/side_effects; `load_cards` synthesizes native cards when cards file empty; rebuild triggers on empty file and on stale mtime vs registry.
|
||||
- [ ] Steps 2–4: red → implement → green.
|
||||
- [ ] Step 5: commit `feat(p1-01): per-tool metadata + capability card synthesis/validation (#24)`
|
||||
|
||||
### Task 2 — select_for_job + verified floor
|
||||
|
||||
**Files:** modify `src/rune/capability_broker.py`.
|
||||
|
||||
- [ ] Step 1: failing tests — coding task returns floor+match set (5–10 tools, never empty); zero-score → floor with reason "floor (no matches)"; reason strings recorded; `find_capability` always present; selection deterministic.
|
||||
- [ ] Steps 2–4: red → implement (`select_for_job`) → green.
|
||||
- [ ] Step 5: commit `feat(p1-01): select_for_job with verified floor (#24)`
|
||||
|
||||
### Task 3 — Operator + runtime wiring (zero-token guarantee)
|
||||
|
||||
**Files:** modify `src/rune/rune_operator.py` (both LocalToolRuntime sites, `_contributor_tools`), `src/rune/local_tool_runtime.py` (find_capability tool + unknown-tool guidance), `src/rune/inference_trace.py` (v3 fields), test file.
|
||||
|
||||
- [ ] Step 1: failing tests — run with selected set exposes ONLY selected schemas (assert `to_ollama_tools` payload size shrinks); unused tools contribute zero tokens; trace v3 fields recorded; unknown tool call returns guidance error; find_capability execution returns matching schema summary and the same run can call the discovered tool.
|
||||
- [ ] Steps 2–4: red → implement → green; full suite regression (baseline 545+84).
|
||||
- [ ] Step 5: commit `feat(p1-01): operator/runtime exposure wiring + find_capability escape hatch + trace v3 (#24)`
|
||||
|
||||
### Task 4 — Composition registration + authority map + docs
|
||||
|
||||
**Files:** `src/rune/orchestration/composition.py` (+ kind), `tests/test_orchestration_composition.py`, `config/v1_authority_map.json` (flip capability_selection paths to LIVE), `docs/capability-selection.md`.
|
||||
|
||||
- [ ] Steps: red → green → map/docs → commit `feat(p1-01): register capability_selection authority + docs (#24)`
|
||||
|
||||
### Task 5 — Canary + CI registration
|
||||
|
||||
**Files:** create `src/rune/canary_p1_01.py`; modify `.gitea/workflows/ci.yml` same commit.
|
||||
|
||||
- [ ] Canary: scripted operator run asserts ~5–10 exposed (not 18), zero-token claim (schema payload byte-size drop), escape hatch round-trip, floor guarantee; prints `P1-01 CAPABILITY AUTHORITY CANARY PASSED`.
|
||||
- [ ] Full suite; ruff; henxels; commit `feat(p1-01): canary + CI registration (#24)`
|
||||
|
||||
## Acceptance (from #24)
|
||||
|
||||
- 100 installed capabilities can exist while a coding job exposes ~5–10 (floor+matches; verified by canary count assertion).
|
||||
- Unused capabilities add zero prompt tokens (schemas excluded from the tools payload; assert byte-size reduction).
|
||||
- Model can dynamically request an unexposed capability (find_capability → summary; unknown-tool error names the hatch).
|
||||
- Cards validated/auto-regenerated; preconditions/risk/side-effect metadata present.
|
||||
|
||||
## Pipeline requirements checklist
|
||||
- [ ] TDD red→green (frozen tests per task)
|
||||
- [ ] ruff check + format on touched/added files
|
||||
- [ ] Spec gate PASS; Quality gate Approved (discharges if needed)
|
||||
- [ ] Full-suite regression green (baseline 545+84)
|
||||
- [ ] Canary green + registered same commit; henxels exit 0
|
||||
- [ ] Closed with formatted comment + PR refs
|
||||
|
||||
## Execution deviations
|
||||
|
||||
1. **Floor name mapping.** Plan floor names `list_dir`/`search_files` do not exist in `tool_executor.TOOLS`; mapped to the actual T1 schema names `list_directory`/`grep_files`. `FLOOR_TOOLS = read_file, write_file, edit_file, list_directory, grep_files` + `find_capability`.
|
||||
2. **find_capability schema + executor live in tool_executor.py.** Plan's Task 3 file list put the hatch only in `local_tool_runtime.py`, but `tool_protocol.validate_tool_request` validates every request against `tool_executor.TOOLS`/`TOOL_INDEX` — a hatch schema outside TOOLS fails normalization before the runtime could dispatch it. Schema, `_find_capability` handler, and `READ_ONLY_TOOLS` entry are in tool_executor; the runtime-side hatch logic (dispatch interception, in-run dynamic exposure, call counting, unknown-tool guidance) is in local_tool_runtime as planned.
|
||||
3. **Dynamic-exposure channel narrowed (frozen-test conflict).** Design Decision 3 said the unknown-tool error path "dynamically registers a requested tool into the running session's tool set when it exists". Frozen test `test_local_tool_runtime.py::test_runtime_rejects_globally_valid_tool_outside_its_allowlist` pins rejection of a registry-valid tool outside the allowlist (P0-05 gating contract). Resolution: the unknown-tool path keeps the rejection and adds `find_capability` guidance; dynamic exposure happens exclusively through `find_capability` execution results (still satisfies the same-run discovery requirement and the acceptance bullet).
|
||||
4. **Frozen tests updated for the planned schema bump v2→v3.** `tests/test_p1_06_model_routing.py` (assertion `schema_version == 2` → `== INFERENCE_TRACE_SCHEMA_VERSION`, test renamed `..._schema_v3`) and `src/rune/canary_p1_06.py` (expects 3). Additive v3 fields keep v1/v2 rows and readers compatible.
|
||||
5. **Task-family source.** `infer_task_family` lives in `rune_operator.py:55`, not `model_routing.py` as the plan parenthetical says; the broker lazy-imports it from rune_operator (degrade → `"general"`).
|
||||
6. **Task 2 test refinement during red→green.** Initial draft asserted exact-set equality for coding queries; the plan spec allows floor+matches (5–10), so tests assert floor ⊆ selection, hatch present, size bounds, and exact floor+hatch only for zero-score queries. Plugin cards cannot add tool schemas (plugins have no executor): a test pins that selection never invents non-schema tools; plugin matches surface through `find_capability` results instead.
|
||||
7. **Resume mechanism.** Global constraint "resume round-trips must use the same selected set" is implemented by persisting an additive `exposed_tools` key into `pending_state` (selected set + any in-run discovered tools); `_runtime_for_model` rebuilds exactly that set, with deterministic re-selection from the task description as fallback for pre-#24 pending states.
|
||||
8. **Composition signature.** `compose_capability_selection(root, state_dir)` mirrors `compose_model_routing`; `CapabilityBroker` accepts `cards_dir` for symmetry but card I/O stays on the memory_env-derived module paths.
|
||||
9. **Cards file rewrite on first load.** `load_cards` now rebuilds the packaged empty `[]` index (missing-native/empty/stale/corrupt triggers) and persists plugin + synthesized native cards. Repo copy untouched; runtime writes go to `RUNE_MEMORY_DIR`.
|
||||
10. **trace v3 writer.** Operator passes `schema_version=INFERENCE_TRACE_SCHEMA_VERSION` (imported) instead of the literal 3, and calls `_capability_selection` (cached per description) for `selected_tools`/`capability_reason`; `find_capability_calls` counted from `response.tool_events`.
|
||||
|
||||
## Completion (P1-01 #24)
|
||||
|
||||
- **Branch:** `p1-01-capability-authority` @ `2db0c3d` base. **Head after plan-tail commit:** see git log below.
|
||||
- **Per-task commits:**
|
||||
- Task 1 `a48d8b1` — feat(p1-01): per-tool metadata + capability card synthesis/validation (#24)
|
||||
- Task 2 `eb88304` — feat(p1-01): select_for_job with verified floor (#24)
|
||||
- Task 3 `185b9f1` — feat(p1-01): operator/runtime exposure wiring + find_capability escape hatch + trace v3 (#24)
|
||||
- Task 4 `8ec9f1a` — feat(p1-01): register capability_selection authority + docs (#24)
|
||||
- Task 5 `e28d1a5` — feat(p1-01): canary + CI registration (#24)
|
||||
- **Red evidence (TDD):**
|
||||
- Task 1: `6 failed, 1 passed` — no risk/preconditions/side_effects on TOOLS, no native synthesis, no rebuild-on-empty/stale.
|
||||
- Task 2: `11 failed, 7 passed` — no `FLOOR_TOOLS`/`select_for_job`/degrade path.
|
||||
- Task 3: `9 failed` — no trace v3 fields, no `_find_capability` executor, no operator `tools=` wiring, hard-fail without hatch guidance.
|
||||
- Task 4: `4 failed, 32 passed` — no `capability_selection` authority kind, map not flipped, no docs/capability-selection.md.
|
||||
- Task 5: canary file absent + unregistered in ci.yml (henxels `ci_runs_all_canaries` bites) → green prints `P1-01 CAPABILITY AUTHORITY CANARY PASSED`.
|
||||
- **Final counts:** `581 passed, 84 subtests passed` (baseline 545 + 84; +36 new frozen tests, zero regressions).
|
||||
- **Checks:** ruff check PASS on all touched files; ruff format PASS on added files; henxels `check --all --plain` exit 0 (21 pre-existing duplicate warnings); all 22 repo canaries green including new `canary_p1_01.py` registered in `.gitea/workflows/ci.yml` same commit.
|
||||
- **Not done (out of scope per brief):** push, PR, spec/quality gate comments — controller owns those steps.
|
||||
|
||||
## Quality discharges (post-review, P1-01 #24)
|
||||
|
||||
All findings from the quality review (Approved WITH findings) discharged on branch `p1-01-capability-authority`. Tests written red-first for behavior changes; deletions/atomicity got targeted tests where meaningful.
|
||||
|
||||
1. **IMPORTANT — duplicate `load_cards` (F811, capability_broker.py:474/:577).** Fix: deleted the FIRST definition (the byte-identical copy at :577 remains, preceded by `build_cards`); module now defines `load_cards` exactly once. Test: `test_load_cards_is_defined_once` — `inspect.getsource` counts `def load_cards` == 1.
|
||||
2. **minor — unused `examples` (capability_broker.py:427).** Fix: deleted the dead `examples = [f"{name} {description}"][:1]` line in `_native_tool_cards` (native cards use empty `semantic_examples` + derived `searchable_text`, unchanged). Test: covered by ruff F841-free check on touched file + existing card-metadata tests (`test_native_cards_carry_tool_metadata`) staying green.
|
||||
3. **minor — `read_persona` schema lost `{"type":"object","properties":{}}` (tool_executor.py:364).** Fix: restored the valid JSON-schema object shape. Test: `test_every_tool_schema_is_a_valid_json_object_schema` — every TOOLS entry has `parameters.type == "object"` and dict `properties` (red against the old `{}`).
|
||||
4. **minor — hatch dispatch bypassed `dispatch_validated_tool` (local_tool_runtime.py:521).** Fix: routed find_capability through the SAME `dispatch_validated_tool` as all other tools (validated dispatch requires only schema-validated args, which the hatch schema defines — no docstring exception needed). Test: `test_hatch_dispatch_routes_through_validated_dispatch` — monkeypatched dispatch spy asserts `find_capability` rides the validated path, call counter still increments, `repo_map` exposure still flows. RED verified against pre-fix code (git stash) before green. Note: `dispatch_validated_tool` receives only (request, executor); boundedness of hatch output stays enforced at the source (`find_capability(top_k=5)`, truncated summary lines) and remains pinned by the existing `test_select_for_job_*` bounds tests.
|
||||
5. **minor — pending_state did not carry the exposed set.** Fix: `_run_loop` now embeds additive `exposed_tools` (sorted `self.tool_names` = selected + dynamically exposed) into the runtime-built pending_state; `resume()` reads the key (fail-closed type check, backward compatible — pre-#24 states lack it) and `_expose_tools` re-adds only names with real schemas before the allowlist check. Test: `test_resume_honors_exposed_tools_in_pending_state` — REAL `LocalToolRuntime` (unmocked), real broker-backed hatch query exposes `git_commit`, run suspends on approval, a FRESH runtime resumes the same state to completion. RED verified against pre-fix code.
|
||||
6. **minor — triple `_capability_selection` per run (rune_operator.py:1795).** Fix: hoisted to a single run-level call (`capability_selection = self._capability_selection(task.description)` before the tier loop); runtime construction and both trace v3 fields read the hoisted value. Operator-level degrade now propagates `reason: "fallback full toolset (broker unavailable)"` so `capability_reason` is never silently empty while the full toolset is selected. Tests: `test_capability_selection_degrade_reason_propagates` (exact reason string + full-TOOLSET selection under broker failure) and `test_operator_run_selects_capabilities_once` (spy asserts `call_count == 1` across a full scripted `send_task` run).
|
||||
7. **minor — non-atomic cards rebuild write (capability_broker.py:497).** Fix: rebuild now writes a sibling temp file (`capability_cards.json.tmp`) then `os.replace` into place; OSError cleanup removes the temp. Test: `test_load_cards_rebuild_write_is_atomic` — monkeypatched `os.replace` spy asserts exactly one temp+rename sequence with a sibling temp path and the final index parses to the loaded cards.
|
||||
|
||||
**Verification:** full suite `588 passed, 84 subtests passed` (baseline 581+84; +7 discharge tests, zero regressions); ruff check PASS on all touched files; ruff format PASS on the touched test file; henxels `check --all --plain` exit 0; `canary_p1_01.py` prints `P1-01 CAPABILITY AUTHORITY CANARY PASSED`.
|
||||
@@ -1,127 +0,0 @@
|
||||
# P1-02 AttentionCapsule → Real Model Product — Implementation Plan (#25)
|
||||
|
||||
> **For agentic workers:** execution contract — FROZEN TESTS ARE THE SPEC. Implement verbatim; record every deviation in the plan tail. TDD red → green. Do not create/complete tasks; controller owns task state.
|
||||
|
||||
**Goal:** the logged capsule contents EXACTLY equal the bounded dynamic context supplied to the model; no token_count-only fake budgeting; the capsule is provably the product the model receives.
|
||||
|
||||
**Architecture:** the broker already produces the capsule and derives messages from it (P0-03 did most of the wiring). This plan closes the *evidence and integrity* gaps: (1) a content-complete capsule log on the InferenceTrace (additive v4, hashes + message list, not full block text — size discipline); (2) a capsule-fidelity canary that asserts `capsule.to_messages()` == the messages passed to `runtime.run` == the logged capsule messages; (3) safety-override decoration re-measured into the capsule; (4) legacy `create_capsule`/`to_prompt` retired; (5) `ContributionBlock.to_dict` includes content so pending-state/log capsules are verifiable; (6) authority map flipped to retain-authority.
|
||||
|
||||
**Tech stack:** stdlib + existing house modules. No new services, no parallel assembly.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Acceptance criterion (from #25): logged capsule contents EXACTLY equal the bounded dynamic context supplied to model. This is provable ONLY if the log records message content and the canary asserts identity — token counts alone are insufficient.
|
||||
- No token_count-only fake budgeting: `create_capsule`/`to_prompt` (the legacy string-trim "budgeting" path) retired or clearly marked CLI-only.
|
||||
- Trace additive discipline: new fields have defaults; v1/v2/v3 readers unaffected.
|
||||
- Degrade-not-crash: capsule logging must never fail the turn (P0-06).
|
||||
- Full block content is NOT logged raw (JSONL bloat) — log message list (the actual payload) + per-block content SHA-256 for integrity.
|
||||
- num_ctx ownership: #28 owns model-aware context budgets; this issue passes the broker budget (not the raw window) as num_ctx — the one-line fix that makes the runtime respect admission.
|
||||
- Safety-override decorated user block MUST be re-measured so `capsule.token_count` stays truthful post-decoration.
|
||||
- TDD; ruff check + format on touched/added files; canary registered in ci.yml same commit.
|
||||
|
||||
## Design Decisions (settled)
|
||||
|
||||
1. **Capsule log location:** InferenceTrace additive v4 fields — `capsule_messages: list[dict] = []` (the exact messages sent), `capsule_content_hashes: dict[str,str] = {}` (block name → SHA-256 of content), `capsule_token_count: int = 0` (post-decoration re-measured total). No separate JSONL file; rotation already handled.
|
||||
2. **Equality invariant:** `capsule.to_messages()` result is captured BEFORE `runtime.run` and stored on the trace; a frozen test asserts trace.capsule_messages == the list passed to runtime.run (identity, not re-measure). The canary asserts the same against a scripted runtime.
|
||||
3. **Safety-override re-measure:** after override mutates the user block content, re-run `estimate_tokens` on the mutated block, update `block.tokens` + `capsule.token_count` + the trace. No re-admission (override grows ≤ ~100 tokens; admission already left headroom).
|
||||
4. **num_ctx:** pass `capsule.token_count + completion_reserve` where completion_reserve = the broker's reserved completion room (window − budget). One-line change; #28 owns the broader model-aware budget work.
|
||||
5. **Runtime fallback tool prompt:** formally exempt — `_with_fallback_prompt` only fires for non-native models, and its cost is outside broker scope. Add a comment + log-only accounting note. Full integration is a #28/#34 follow-up.
|
||||
6. **Legacy retirement:** `create_capsule` and `to_prompt` marked deprecated (docstring + `warnings.warn(DeprecationWarning)`), zero live callers verified by test. `to_dict` gains `content` field. Authority map: `attention_capsule.py` → retain-authority under ContextBroker.
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1 — ContributionBlock.to_dict includes content + AttentionCapsule message capture
|
||||
|
||||
**Files:** modify `src/rune/context_broker.py` (to_dict + capsule), `src/rune/attention_capsule.py` (deprecation), test `tests/test_p1_02_capsule_authority.py`.
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `ContributionBlock.to_dict()` includes `"content"`; `AttentionCapsule.capture_messages()` → stores the exact list on `capsule.sent_messages` (not a dataclass field — set post-hoc like the existing dynamic attrs); `create_capsule`/`to_prompt` emit DeprecationWarning.
|
||||
|
||||
- [ ] **Step 1: failing tests** — to_dict includes content matching the block's content; to_dict without content key in old fixture still loads (back-compat); capture_messages stores the list; to_messages re-derivation equals captured list for the same capsule; create_capsule raises DeprecationWarning.
|
||||
- [ ] **Step 2: run → red**
|
||||
- [ ] **Step 3: implement**
|
||||
- [ ] **Step 4: green**
|
||||
- [ ] **Step 5: commit** `feat(p1-02): capsule content logging + message capture + legacy deprecation (#25)`
|
||||
|
||||
### Task 2 — Safety-override re-measure + num_ctx budget fix
|
||||
|
||||
**Files:** modify `src/rune/rune_operator.py`, test file.
|
||||
|
||||
- [ ] **Step 1: failing tests** — after safety-override mutates the user block, `capsule.token_count` reflects the new estimate (not stale); trace `per_block_tokens["user"]` matches; `num_ctx` passed to runtime.run equals `capsule.token_count + completion_reserve` (not the raw window).
|
||||
- [ ] **Step 2: red** → **Step 3: implement** → **Step 4: green**
|
||||
- [ ] **Step 5: commit** `feat(p1-02): safety-override re-measure + num_ctx budget propagation (#25)`
|
||||
|
||||
### Task 3 — Trace v4 capsule fields + operator logging
|
||||
|
||||
**Files:** modify `src/rune/inference_trace.py` (v4 fields), `src/rune/rune_operator.py` (log messages + hashes into trace), test file.
|
||||
|
||||
**Interfaces:** Consumes Task 1. Produces: `InferenceTrace.capsule_messages`, `capsule_content_hashes`, `capsule_token_count` (additive, defaults preserve v1–v3 readers).
|
||||
|
||||
- [ ] **Step 1: failing tests** — trace record carries capsule_messages equal to the messages the scripted runtime received; content hashes present and correct; old trace rows without v4 fields still parse.
|
||||
- [ ] **Step 2: red** → **Step 3: implement** → **Step 4: green**; full suite regression (baseline 588+84).
|
||||
- [ ] **Step 5: commit** `feat(p1-02): trace v4 capsule fidelity fields (#25)`
|
||||
|
||||
### Task 4 — Authority map + docs
|
||||
|
||||
**Files:** `config/v1_authority_map.json` (attention_capsule → retain-authority, evidence), `tests/test_v1_authority_map.py` (if pinned), `docs/capsule-fidelity.md` (create: what the capsule is, the equality invariant, how to verify, num_ctx propagation).
|
||||
|
||||
- [ ] Steps: map update → docs → commit `feat(p1-02): capsule authority map + docs (#25)`
|
||||
|
||||
### Task 5 — Canary + CI registration
|
||||
|
||||
**Files:** create `src/rune/canary_p1_02.py`; modify `.gitea/workflows/ci.yml` same commit.
|
||||
|
||||
- [ ] Canary: scripted operator run asserts (1) trace.capsule_messages == runtime-received messages (exact list identity), (2) capsule_token_count truthful post-decoration, (3) num_ctx == budget not window, (4) content hashes match block contents, (5) no DeprecationWarning suppressed. Prints `P1-02 CAPSULE FIDELITY CANARY PASSED`.
|
||||
- [ ] Full suite; ruff; henxels; commit `feat(p1-02): canary + CI registration (#25)`
|
||||
|
||||
## Acceptance (from #25)
|
||||
|
||||
- Logged capsule contents EXACTLY equal the bounded dynamic context supplied to model — proven by trace v4 `capsule_messages` == `runtime.run` messages, asserted in frozen test + canary.
|
||||
- No token_count-only fake budgeting — `create_capsule`/`to_prompt` deprecated, zero live callers verified; admission remains the real budget enforcement.
|
||||
|
||||
## Pipeline requirements checklist
|
||||
- [x] TDD red→green (frozen tests per task)
|
||||
- [x] ruff check + format on touched/added files
|
||||
- [ ] Spec gate PASS; Quality gate Approved (discharges if needed)
|
||||
- [x] Full-suite regression green (baseline 588+84 → 604+84, +16 new)
|
||||
- [x] Canary green + registered same commit; henxels exit 0
|
||||
- [ ] Closed with formatted comment + PR refs
|
||||
|
||||
## Completion Record (P1-02 implementer, 2026-09-06)
|
||||
|
||||
**Status:** all 5 tasks complete. Branch `p1-02-capsule-authority`, head `96e42ae`.
|
||||
|
||||
**Commits:**
|
||||
1. `d0550b6` feat(p1-02): capsule content logging + message capture + legacy deprecation (#25)
|
||||
2. `ec0cf7a` feat(p1-02): safety-override re-measure + num_ctx budget propagation (#25)
|
||||
3. `8ff6a83` feat(p1-02): trace v4 capsule fidelity fields (#25)
|
||||
4. `90bcd59` feat(p1-02): capsule authority map + docs (#25)
|
||||
5. `96e42ae` feat(p1-02): canary + CI registration (#25)
|
||||
|
||||
**Final counts:** 604 passed + 84 subtests (baseline 588+84; +16 new frozen tests, 0 regressions).
|
||||
Canary: `P1-02 CAPSULE FIDELITY CANARY PASSED`. henxels exit 0 (21 pre-existing warnings).
|
||||
|
||||
**Red evidence** (initial run of tests/test_p1_02_capsule_authority.py, 14 failed / 2 passed):
|
||||
`test_contribution_block_to_dict_includes_content`, `test_capture_messages_stores_the_exact_list`,
|
||||
`test_create_capsule_emits_deprecation_warning`, `test_safety_override_reremeasures_capsule_token_count`,
|
||||
`test_num_ctx_is_budget_plus_completion_reserve_not_window`, `test_inference_trace_schema_v4_fields`,
|
||||
`test_operator_trace_record_carries_capsule_messages` (+7 more) — all red before their task's implementation, green after.
|
||||
|
||||
**Deviations from plan:**
|
||||
1. **Schema v4 bump touched pinned literals:** `tests/test_p1_01_capability_authority.py` (3 assertions) and canaries
|
||||
`canary_p1_01.py` / `canary_p1_06.py` pinned `schema_version == 3`. Updated those pins to 4 in the same Task-3 commit —
|
||||
the additive discipline protects v1–v3 ROWS, not the literal; `docs/inference-trace.md` schema table updated to v4 with a schema history note.
|
||||
2. **trace["blocks"] user-entry re-measure:** the frozen test demanded `per_block_tokens["user"]` match post-decoration; the
|
||||
re-measure therefore also updates the admitted user entry in the broker trace dict (not only `block.tokens` + `capsule.token_count`).
|
||||
3. **num_ctx degrade path:** added a `num_ctx <= 0` fallback to the discovered window for stand-in brokers (trace missing
|
||||
window/budget keys) — degrade-not-crash, beyond the literal one-liner, pinned by `test_num_ctx_falls_back_to_window_when_budget_basis_missing`.
|
||||
4. **Trace v4 extraction is Mock-hardened:** pre-existing frozen tests construct the capsule as a `Mock`; the operator's
|
||||
hash/message extraction guards non-list/non-dict content and falls back gracefully (no fidelity evidence on stand-in capsules).
|
||||
5. **`capture_messages` deep-copies** the sent list (test-pinned) so caller-side mutation can never diverge the logged payload.
|
||||
|
||||
**Reviewer flags:**
|
||||
- `InferenceTrace.capsule_messages` logs the FULL message payload per run — accepted per plan (message list, not raw blocks);
|
||||
watch JSONL rotation headroom if history blocks grow (#28 compaction bounds this).
|
||||
- `docs/inference-trace.md` "Record schema (version 4)" supersedes the version-1 table heading; no other docs referenced v3.
|
||||
- Authority map: `attention_capsule.py` entry now `retain-authority` under ContextBroker (context_construction);
|
||||
`test_v1_authority_map.py` green (path existence, disposition vocabulary).
|
||||
@@ -1,155 +0,0 @@
|
||||
# P1-06 Model/Provider Routing — Implementation Plan (#29)
|
||||
|
||||
> **For agentic workers:** execution contract — FROZEN TESTS ARE THE SPEC. Implement verbatim; record every deviation in the plan tail. TDD red → green. Do not create/complete tasks; controller owns task state.
|
||||
|
||||
**Goal:** one authoritative `ModelRouter` + `ProviderAdapter` interface; deterministic → tiny local → strong local → cloud escalation driven by verification evidence, with routing reason and cost visible, and provider removal never erasing state.
|
||||
|
||||
**Architecture:** a new `src/rune/model_routing.py` module (single file, house forge-Protocol idiom) owning `ProviderAdapter` (runtime-checkable Protocol), `OllamaAdapter` (first adapter, delegating to `OllamaChatTransport`), `RoutingCandidate`/`MeasuredModelProfile` (registry rows), and `ModelRouter.rank()` (advisory ranked selection in the `orchestration/router.py` idiom — never mutating inputs). The operator tier loop consults the router for each attempt and records the reason; the trace schema gains additive optional fields (v2, backward-compatible readers); `composition.py` registers `model_routing` as the single authority.
|
||||
|
||||
**Tech stack:** Python 3.11 stdlib + existing house modules only. No new services; Ollama-first; cloud adapters are declared, not live (no cloud credentials enter this issue).
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- Authority map `model_routing` migration rule verbatim: "One router selects the cheapest verified-capable model from measured profiles and records the reason."
|
||||
- Do NOT integrate the three archive-candidate escalation engines (`elastic_cognition`, `tiered_executor`, `progressive_escalation`) — they stay archive; `should_escalate` may be *borrowed as an idea* only.
|
||||
- `provider_registry._execute_tool_calls` parallel path stays dead (single tool-execution authority: `LocalToolRuntime`, #32).
|
||||
- `num_ctx`/admission stays with ContextBroker + #28; router consumes `context_window` as a capability input only.
|
||||
- Trace schema change must be additive (new optional fields, `schema_version: 2`); existing v1 consumers keep working.
|
||||
- No cloud migration, no new provider services, no credentials in repo.
|
||||
- TDD: every task's frozen tests written first, shown failing, then green.
|
||||
- `ruff check` + `ruff format --check` on touched/added files; canary registered in `ci.yml` same commit as the canary file.
|
||||
|
||||
## Design Decisions (settled)
|
||||
|
||||
1. **Router authority boundary:** the router is a pure ranked-candidate advisor; the **operator tier loop is the final decider** (it already owns attempts). "Verified-capable" pre-#33 = the model's measured profile from `inference_traces.jsonl` (success rate per task_family) + static capabilities (`supports_tools`, `context_window` ≥ task need). No OutcomeLedger dependency yet — #33 adopts later.
|
||||
2. **Escalation semantics:** within a run, the existing safety-tier loop is unchanged. **Cross-tier model escalation** keys off the run-level result: when `success=False` (or tool-failure) on a local model, the NEXT run for that task family may start at a higher model tier (deterministic → strong local → cloud-when-configured). Router never burns cloud tokens mid-run; escalation is a start-tier decision + recorded reason. `provider_budget.ProviderBudgetTracker` gates cloud candidates when present.
|
||||
3. **Registry substrate:** new measured registry (`MeasuredModelProfile`) built from `inference_traces.jsonl` (already persisted, append-only) keyed by `(provider, model)` with per-task-family success/latency/cost aggregates; `model_registry.ModelProfile` static ladder provides the deterministic capability floor (tier ordering, supports_tools). Two inputs, one router — no third store.
|
||||
4. **Trace schema:** bump `schema_version` to 2 with additive optional fields `routing_reason: str = ""`, `routing_mode: str = ""`, `task_family: str = ""`; `_fold_attempt` keeps per-run folding (reason = the start-tier decision reason).
|
||||
5. **ProviderAdapter scope: chat-only.** One `chat()`-shaped Protocol delegating to existing transports (`OllamaAdapter` wraps `OllamaChatTransport`). Capability discovery stays in `local_tool_runtime` (#28 boundary). OpenAI/Anthropic adapters are declared but `NotImplementedAvailable=False` (raise `ProviderUnavailable`), keeping the interface honest without live cloud code.
|
||||
6. **State durability:** measured profiles persist to `RUNE_HOME/state/model_routing_profiles.json` (same pattern as model_profiles), keyed by provider+model; adapter configs (URLs) read from env/config each boot. Disconnecting a provider removes only the adapter instance — profiles and outcomes stay on disk.
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1 — ProviderAdapter Protocol + OllamaAdapter
|
||||
|
||||
**Covers:** provider_adaptation migration rule; authority-map `provider_adaptation` → live.
|
||||
|
||||
**Files:**
|
||||
- Create: `src/rune/model_routing.py`
|
||||
- Test: `tests/test_p1_06_model_routing.py`
|
||||
|
||||
**Interfaces:**
|
||||
- Produces: `ProviderAdapter` (Protocol: `name`, `available() -> bool`, `chat(model, messages, tools, options, think) -> ChatResponse-shaped dict`), `OllamaAdapter(transport=None)`, `ProviderUnavailable(RuntimeError)`.
|
||||
|
||||
- [ ] **Step 1: failing tests**
|
||||
|
||||
```python
|
||||
def test_provider_adapter_protocol_shape():
|
||||
from rune.model_routing import ProviderAdapter, OllamaAdapter, ProviderUnavailable
|
||||
a = OllamaAdapter(transport=_FakeTransport())
|
||||
assert isinstance(a, ProviderAdapter)
|
||||
assert a.name == "ollama"
|
||||
with pytest.raises(ProviderUnavailable):
|
||||
_CloudStubAdapter().chat(model="x", messages=[], tools=None, options={}, think=False)
|
||||
|
||||
def test_ollama_adapter_delegates_to_transport():
|
||||
a = OllamaAdapter(transport=_FakeTransport(expected={"message": {"content": "hi"}}))
|
||||
resp = a.chat(model="qwen3:8b", messages=[{"role": "user", "content": "hello"}], tools=None, options={}, think=False)
|
||||
assert resp["message"]["content"] == "hi"
|
||||
assert a.available() is True
|
||||
```
|
||||
|
||||
- [ ] **Step 2: run → red** (`ModuleNotFoundError: rune.model_routing`)
|
||||
- [ ] **Step 3: implement** Protocol + OllamaAdapter delegating to `OllamaChatTransport.chat()`; `available()` pings `/api/tags` with 2s timeout, cached 30s; `CloudStubAdapter.chat` raises `ProviderUnavailable` (cloud adapters land when credentials/config exist — never silently fake).
|
||||
- [ ] **Step 4: green**
|
||||
- [ ] **Step 5: commit** `feat(p1-06): ProviderAdapter protocol + OllamaAdapter (#29)`
|
||||
|
||||
### Task 2 — Measured registry + ModelRouter.rank()
|
||||
|
||||
**Files:** modify `src/rune/model_routing.py`; test as Task 1.
|
||||
|
||||
**Interfaces:** Produces `MeasuredModelProfile` dataclass, `ModelRouter(candidates, profiles_path)` with `.rank(task_family, need_tools, min_context) -> list[RoutingDecision]`; `RoutingDecision(model, provider, tier, reason, estimated_cost)`.
|
||||
|
||||
- [ ] **Step 1: failing tests** — deterministic tier floor respected (`supports_tools=False` excluded when `need_tools`); cheapest verified-capable first; reason string names the evidence (`"measured 12/13 success task_family=code"` / `"capability-floor tier1"`); profiles load from a temp `state/model_routing_profiles.json`; missing profiles file → capability-floor ordering, reason `"no-measured-profiles"`.
|
||||
- [ ] **Step 2: red** → **Step 3: implement** (aggregate traces input is a list of dict-shaped trace rows; keep pure — no I/O inside `rank()`; a `load_profiles(path)` helper does I/O) → **Step 4: green**
|
||||
- [ ] **Step 5: commit** `feat(p1-06): measured model registry + ModelRouter.rank (#29)`
|
||||
|
||||
### Task 3 — Operator wiring + escalation start-tier + trace fields
|
||||
|
||||
**Files:** modify `src/rune/rune_operator.py`, `src/rune/inference_trace.py`, `src/rune/rune_tui.py` (`_get_model_name` delegate), test file.
|
||||
|
||||
**Interfaces:** Consumes Tasks 1–2. Produces: `_execute_on_model` builds `LocalToolRuntime(transport=router_selected.transport)`; `InferenceTrace` gains `routing_reason/routing_mode/task_family` (schema_version 2, defaults preserve v1 readers); `_get_model_name("cloud")` returns router's cloud candidate or falls back with recorded reason (kills the "cloud=qwen2.5:7b" lie); `_decide_model` archived (router replaces keyword heuristic).
|
||||
|
||||
- [ ] **Step 1: failing tests** — scripted router + scripted transport: a run whose start-tier decision came from the router records `routing_reason` in the trace; `schema_version == 2`; v1-row reader still parses (back-compat test on a fixture line without new fields); `_get_model_name("cloud")` with no cloud adapter returns the recorded fallback reason.
|
||||
- [ ] **Step 2: red** → **Step 3: implement** → **Step 4: green** (full suite regression: baseline 526+84)
|
||||
- [ ] **Step 5: commit** `feat(p1-06): wire ModelRouter into operator + trace v2 routing fields (#29)`
|
||||
|
||||
### Task 4 — Composition registration + authority map + docs
|
||||
|
||||
**Files:** modify `src/rune/orchestration/composition.py` (AUTHORITY_KINDS + `compose_model_routing`), `tests/test_orchestration_composition.py` (extend invariant), `config/v1_authority_map.json` (model_routing/provider_adaptation paths → LIVE with new evidence lines), `docs/model-routing.md` (create: ladder, escalation policy, cost visibility, provider-removal guarantee).
|
||||
|
||||
- [ ] Steps: extend composition test red → implement compose function green → authority-map evidence update → docs → commit `feat(p1-06): register model_routing authority + docs (#29)`
|
||||
|
||||
### Task 5 — Canary + CI registration + closing
|
||||
|
||||
**Files:** create `src/rune/canary_p1_06.py`; modify `.gitea/workflows/ci.yml` (same commit).
|
||||
|
||||
- [ ] Canary: scripted adapter + router; asserts rank order, reason recording, trace v2 fields, provider-removal profile persistence (delete adapter object, profiles file intact); prints `P1-06 MODEL ROUTING CANARY PASSED`. Registered in ci.yml same commit.
|
||||
- [ ] Full suite regression green; ruff clean; commit `feat(p1-06): canary + CI registration (#29)`
|
||||
|
||||
## Acceptance (from #29)
|
||||
|
||||
- Same task can route among local tiers without changing agent identity (router returns candidates; operator identity/loop unchanged).
|
||||
- Routing reason and cost visible: `RoutingDecision.reason` + `estimated_cost` land in the trace and the response envelope's degraded/routing observability.
|
||||
- Provider removal does not erase state: profiles/outcomes live in rune-home JSON keyed by provider+model, independent of adapter instances.
|
||||
- Verification-driven: start-tier escalation keys off measured run results (`success`/tool-failure in traces), never model self-report.
|
||||
|
||||
## Pipeline requirements checklist
|
||||
- [ ] TDD red→green (frozen tests per task)
|
||||
- [ ] ruff check + format on touched/added files
|
||||
- [ ] Spec gate PASS (independent reviewer)
|
||||
- [ ] Quality gate Approved (independent reviewer)
|
||||
- [ ] Full-suite regression green (baseline 526 + new frozen tests)
|
||||
- [ ] Canary green + registered in ci.yml same commit
|
||||
- [ ] henxels exit 0
|
||||
- [ ] Closed with formatted comment + PR refs
|
||||
|
||||
## Execution deviations
|
||||
|
||||
1. **Task 1 — `OllamaAdapter.available()` transport probe:** the frozen test's `_FakeTransport` carries its own `available()` and the test asserts `available() is True` without any HTTP. Implementation consults a transport-provided probe when present (and the transport has no `base_url`); production `OllamaChatTransport` has no such probe, so `/api/tags` + 2s timeout + 30s cache behavior is unchanged.
|
||||
2. **Task 2 — reason string is `no-measured-profiles tier<N>`, not `no-measured-profiles`:** the plan's example quoted `"no-measured-profiles"` as the missing-file reason; the frozen tests were written with a tier suffix (consistent with the `capability-floor tierN` example). Recorded here since the plan text names the bare string.
|
||||
3. **Task 2 — semantics clarified by frozen tests:** (a) when measured profiles exist, only verified-capable (measured, non-zero-success) candidates are ranked — unmeasured candidates are excluded rather than ranked with a `capability-floor` reason; (b) measured verification overrides the static `supports_tools`/context claim (a model that measurably succeeds in a family is eligible for it); (c) the measured profile's `estimated_cost_usd` overrides the static floor cost (this is what makes "cheapest verified-capable" meaningful — T4's static entry is paid, 15.0, and free measured local models win).
|
||||
4. **Task 3 — `_decide_model` archived by rename, not deletion:** the plan says "_decide_model archived (router replaces keyword heuristic)". Implemented as `_fallback_model` with an ARCHIVED docstring; the keyword body survives only on the router-unavailable degrade path (P0-06 degrade-not-crash) — deleting it outright would have made router failure a hard task failure.
|
||||
5. **Task 3 — `task_family` inference added:** the plan lists `task_family` as a trace field but assigns no owner; implemented `infer_task_family()` (keyword map, documented as the same evidence basis as the archived `_decide_model` heuristic, used only as a family label). Frozen tests encode `code` for a code-family description.
|
||||
6. **Task 3 — TUI cloud fallback tested at module level:** `_get_model_name` cannot run without a constructed Textual app (reactives), so resolution logic moved to module-level `_resolve_model_name()` and the method delegates. The frozen-test fallback string is `"cloud-unavailable-fallback: ..."`; the router's declared cloud candidate is NOT returned because `CloudStubAdapter.available()` is False (declared-not-live) — this is what kills the "cloud=qwen2.5:7b" lie without introducing a new one.
|
||||
7. **Task 4 — `compose_model_routing` returns a `ModelRoutingAuthority` handle** and registers the router under the new `model_routing` authority kind; composition test extended per plan.
|
||||
8. **Schema-version count:** existing `test_p0_07_inference_trace.py` asserted `schema_version == 1` in `_make_trace` (a v1-shape helper) and in one rotation test over v1 fixture rows — those tests still pass unchanged because the v1 helpers construct their own records and `read_traces` is schema-agnostic. No test edits were needed; `INFERENCE_TRACE_SCHEMA_VERSION` and the operator's trace emission are now 2.
|
||||
|
||||
## Completion record
|
||||
|
||||
**Branch:** `p1-06-model-routing` (head `9dcbdbd2fe58fe814aba6e9ddcb27059337d018d`), based on master `451fbf5`.
|
||||
|
||||
**Per-task commits:**
|
||||
|
||||
| Task | Commit | Subject |
|
||||
|------|--------|---------|
|
||||
| 1 | `bea80b0` | feat(p1-06): ProviderAdapter protocol + OllamaAdapter (#29) |
|
||||
| 2 | `4358e5d` | feat(p1-06): measured model registry + ModelRouter.rank (#29) |
|
||||
| 3 | `bb416ad` | feat(p1-06): wire ModelRouter into operator + trace v2 routing fields (#29) |
|
||||
| 4 | `c668eb4` | feat(p1-06): register model_routing authority + docs (#29) |
|
||||
| 5 | `9dcbdbd` | feat(p1-06): canary + CI registration (#29) |
|
||||
|
||||
**Final test counts:** full suite **539 passed + 84 subtests** (baseline 526 + 84; +11 new frozen P1-06 tests, +2 composition/authority-map tests net of refactor). P1-06 frozen tests: 13 passing (11 in Task-1/2/3 sections + 2 Task-4 tests). Canary: `P1-06 MODEL ROUTING CANARY PASSED` (registered in ci.yml in commit `9dcbdbd`, same commit as the canary file). henxels `check --all --plain`: exit 0 (21 pre-existing warnings). ruff check clean on all touched files; ruff format applied to added files (`model_routing.py`, `test_p1_06_model_routing.py`, `canary_p1_06.py`) — master-touched files (`rune_operator.py`, `rune_tui.py`, `inference_trace.py`, `composition.py`) keep existing style per the CI added-files-only format gate.
|
||||
|
||||
**Evidence:** TDD red was demonstrated for every task (ModuleNotFoundError → 2 failed; rank reason/order assertions → 4 failed; missing routing attrs/`_build_model_router`/`_resolve_model_name` → 3 failed; missing `compose_model_routing`/authority-map paths → 2 failed) before each green implementation.
|
||||
|
||||
## Quality-gate discharges (second review round)
|
||||
|
||||
- BLOCKER: T0 "deterministic" pseudo-model removed from CAPABILITY_FLOOR (not executable by any transport; ranking it made the operator send a 404 model id). Tier numbers sort numerically via _tier_number().
|
||||
- I-2 (measured loop closed): operator run now folds its trace into the profiles store via update_profiles_from_traces + save_profiles (atomic), wired after record_trace with degrade-not-crash guard. Production call site exists; profiles are no longer permanently empty.
|
||||
- I-3 (cloud fallback lie): _resolve_model_name returns a runnable fallback id with the reason surfaced in routing_reason, never a reason-string as a model id.
|
||||
- I-4 (atomicity): save_profiles uses temp + os.replace; load_profiles coerces/drops non-numeric stats.
|
||||
- I-5 (floor semantics): min_context is a hard limit even for measured candidates; measured tool-family success overrides the static supports_tools claim (frozen semantics preserved, evidence visible as "measured ok/n"); unmeasured candidates are always floor-eligible (never-empty family guarantee).
|
||||
- Frozen-test expectation updates (recorded as deviations): rank() may return unmeasured floor candidates AFTER measured ones (never-empty); the measured-success test now asserts 5/5 tool evidence keeps the model ranked.
|
||||
- MINOR-6 partially addressed: router construction is per-send_task (fresh profiles each run); caching deferred to a follow-up note.
|
||||
- MINOR-7/MINOR-8 accepted: canary provider-removal check proves persistence semantics at the store level; negative tests added where material (garbage JSON, atomicity, numeric tier sort).
|
||||
@@ -1,137 +0,0 @@
|
||||
# #122 Failure auto-filing: subsystem failures become forge issues (dedup + spool) — Implementation Plan
|
||||
|
||||
> **Execution contract:** TDD red→green; controller owns task state; conventional commits referencing (#122); PR + protected checks only; deviations recorded in the plan tail.
|
||||
|
||||
## Goal
|
||||
|
||||
When a subsystem fails, Rune files ONE forge issue per failure signature, comments on repeats (`+N occurrences`), spools locally when the forge is unreachable, and reconciles the spool on recovery. T0/T1 failures file via ForgeClient; T2 stays journal-only (cron noise must not become board noise). Silent failure was eliminated in #22; this closes the loop: detect → record → track → fix.
|
||||
|
||||
## Architecture
|
||||
|
||||
- **New module `src/rune/failure_auto_filer.py`:**
|
||||
- `FailureAutoFiler(client: ForgeClient, journal, dedup_window_hours=24)` — hooks `subsystem_health.record_failure` (wraps `registry().record` call sites via a `record_and_maybe_file()` entry point; the registry itself stays untouched — composition, not modification).
|
||||
- Signature = `(subsystem, error_class)` where `error_class` = exception class name or first token of the error string. NOT the full message (variable payloads would flood).
|
||||
- Dedup: before filing, `client.search_issues("[auto-failure:<subsystem>:<error_class>]")` — **server-side title search, NOT client-side list+filter**: `list_open_issues()` is hard-capped at `limit=50` with no pagination (forge_client.py:86-93, verified 2026-09-07), so list+filter silently misses older failure issues and duplicate-files. `search_issues` is a NEW protocol method added to `ForgeClient` (NullAdapter → `[]`, GiteaForgejoAdapter → `/issues/search?type=issues&q=<marker>`). Found → `add_comment(issue_number, "+N occurrences (last: <error excerpt> @ <ts>)")`. Not found → `create_issue` with the marker title, body carrying tier, excerpt, count-1, occurrence history tail.
|
||||
- Spool: on ANY client exception (forge unreachable/auth/5xx), append a JSON line to the existing event journal (`memory/journal/events.jsonl`, already durable) with `kind: "failure_auto_file_spool"`. A `reconcile()` pass (called at operator startup and from maintenance cadence) replays spooled lines: dedup logic re-applies; confirmed-filed lines are marked consumed.
|
||||
- Tier gate: `tier in (0, 1)` → file/comment; `tier >= 2` → journal only. The `in-memory occurrence counter` (per signature, process-lifetime) coalesces bursts: first occurrence files, subsequent occurrences within the dedup window batch into one comment per 10 occurrences or 10 minutes (whichever first) — prevents a 3am crash-loop from writing 200 comments.
|
||||
- **Wiring:** one `_auto_filer` instance on the operator composition root; `subsystem_health.record_failure()` gains an optional `filer` callback param (default None — zero behavior change when unwired); the two existing call-path families (context_broker :313, capability_broker :120, and any others found by grep) pass it through via the `registry().record` path unchanged — the filer observes AFTER the journal record, fire-and-forget (never blocks, never raises into the failure path — degrade-not-crash: filer exceptions are swallowed + journaled).
|
||||
- **Config:** `RUNE_AUTOFILE_OFF=1` env disables entirely (fail-safe off switch); forge endpoint from existing ForgeClient config (`create_forge_client()` — returns NullAdapter when unconfigured, which makes the filer a no-op by construction).
|
||||
|
||||
## Design Decisions (settled)
|
||||
|
||||
1. Marker-in-title dedup via NEW `search_issues` protocol method (server-side search — pagination-safe; list_open_issues' 50-cap makes client-side filtering unsound) — survives issue renames, works on Gitea AND any ForgeClient impl.
|
||||
2. Spool rides the EXISTING event journal (P0-06 durable store) — no new storage subsystem (anti-complexity rule).
|
||||
3. The filer NEVER runs in the exception's hot path synchronously against the network: it enqueues, a single worker drains (thread + queue.Queue, daemon). Failure-path latency stays untouched.
|
||||
4. T2 journal-only is enforced in ONE place (the filer), not at call sites.
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1 — AutoFiler core: signature, dedup, file/comment
|
||||
Failing tests first (fake ForgeClient + fake journal): T0/T1 file once per signature; repeat → comment with count; T2 → no client calls; client exception → spool line written, no raise. Implement → green → commit `feat(failures): auto-filer core with signature dedup (#122)`.
|
||||
|
||||
### Task 2 — Burst coalescing + reconcile
|
||||
Failing tests: 50 rapid occurrences → ≤2 client writes; reconcile replays spool (files missing, marks consumed, dedups against live issues); reconcile with forge still down → spool preserved. Implement → green → commit `feat(failures): burst coalescing + spool reconciliation (#122)`.
|
||||
|
||||
### Task 3 — Wiring + env off-switch + docs
|
||||
`record_failure(filer=...)` param; composition-root wiring; `RUNE_AUTOFILE_OFF`; docs section (failure-handling doc or README ops section). Failing test: unwired call sites behave identically to today; wired sites file. Green → commit `feat(failures): wire auto-filer through composition (#122)`.
|
||||
|
||||
## Acceptance (from #122)
|
||||
|
||||
- One issue per failure signature, repeat occurrences commented with rising count ✓ Tasks 1–2
|
||||
- Forge unreachable → spooled locally, lands on recovery ✓ Task 2
|
||||
- T2 journal-only ✓ Task 1
|
||||
- You get pinged through the forge's own watch system ✓ (consequence of real issue creation)
|
||||
|
||||
## Pipeline requirements checklist (from #122)
|
||||
|
||||
- [ ] TDD red→green per task
|
||||
- [ ] `ruff check`/`format --check` clean
|
||||
- [ ] Spec gate PASS; quality gate Approved
|
||||
- [ ] Full suite green (604+new)
|
||||
- [ ] Existing canaries green
|
||||
- [ ] Closed with formatted comment + PR ref
|
||||
|
||||
## Completion Record
|
||||
|
||||
(status: COMPLETE — 2026-09-07, branch failure-auto-filing, base 2840f71)
|
||||
|
||||
- Commits:
|
||||
- d0df453 feat(failures): auto-filer core with signature dedup (#122)
|
||||
- 082e17d feat(failures): burst coalescing + spool reconciliation (#122)
|
||||
- e5e8727 feat(failures): wire auto-filer through composition (#122)
|
||||
- Counts: 663 passed + 84 subtests (baseline 604 + 84 → +59 tests); red
|
||||
evidence per task recorded in session trace (ModuleNotFoundError /
|
||||
ImportError red runs before each implementation).
|
||||
- Verification: full pytest green; pyright ratchet PASS (118/120 errors,
|
||||
414/465 warnings — below baseline); ruff format --check + E9,F63,F7,F82
|
||||
clean on all changed files; henxels check --all exit 0 (21 pre-existing
|
||||
warnings, no canary added, ci.yml untouched).
|
||||
- Frozen kill tests: T2 → zero client calls (sync, worker, real call site
|
||||
via record_failure path); forge-down → spool + never raises into the
|
||||
failure path (sync + worker). Both in tests/test_failure_auto_filer.py.
|
||||
- Deviations (recorded per contract):
|
||||
1. ~~Spool consumption = in-place journal line rewrite (`consumed: true`)
|
||||
with a sidecar `<journal>.consumed` fallback when the rewrite fails~~
|
||||
SUPERSEDED by gate discharge finding 2: consumption is the append-only
|
||||
sidecar ONLY — the canonical journal is never rewritten (racy +
|
||||
crash-poisoning; see Gate Discharge).
|
||||
2. Spool records are plain JSON lines co-located in events.jsonl (NOT
|
||||
canonical envelopes): DurableEventStore.append's dialect guard is
|
||||
enforced only on its own append path, so co-located foreign lines
|
||||
never violate the canonical store contract; verified no production
|
||||
consumer reads offsets.
|
||||
3. Batch comment cadence flushes when pending EXCEEDS batch_size
|
||||
(comment at occurrences 11/21...), not exactly at 10/20 — "one
|
||||
comment per 10 occurrences" semantics; first occurrence always files
|
||||
immediately (plan: first occurrence files).
|
||||
4. Worker additionally coalesces the already-queued backlog into one
|
||||
write per signature (50-burst → ≤2 writes acceptance requires it).
|
||||
5. record_failure forwards the ORIGINAL exception to the filer (plan
|
||||
said "hooks record_failure" without specifying the payload); the
|
||||
filer extracts the class name itself — signature stays authoritative.
|
||||
6. rune_operator._wire_auto_filer uses the repo's established dual-mode
|
||||
import pattern (package + script top-level deploy), matching the
|
||||
existing record_failure import sites.
|
||||
7. NullAdapter search/create never raises, so a spool line replayed
|
||||
against NullAdapter is treated as NOT filed and stays spooled (not
|
||||
falsely consumed) — safer under the "never lose a failure" rule.
|
||||
|
||||
## Gate Discharge (2026-09-07 — two gate reviews, branch failure-auto-filing)
|
||||
|
||||
All 12 findings discharged red-first (failing test committed/verified before each
|
||||
fix); O(N²) mark_consumed (finding 12) is resolved by the finding-2 sidecar
|
||||
design (append-only, no rewrite) — recorded here rather than in a separate commit.
|
||||
|
||||
| # | Finding | Commit | Red evidence |
|
||||
|---|---------|--------|--------------|
|
||||
| 1 | search_issues hit nonexistent `/issues/search` (404 → spool forever) | fe9709c | 3 URL tests failed against old client (`assert captured["url"].startswith(..."/issues?")`) |
|
||||
| 2 | mark_consumed read-modify-write on shared canonical journal (racy, crash-poisoning) | 38b942c | byte-identity + concurrent-append tests failed (rewrite happened / append destroyed) |
|
||||
| 3 | reconcile per-line replay → comment flood after outage | 6765a78 | 10 spooled lines → 10 writes (`assert len(created)+len(commented) == 1` failed with 10) |
|
||||
| 4 | no secret redaction into persistent issues | 9105ca8 (+fc103fb test fix) | ImportError: `redact` missing |
|
||||
| 5 | reconcile bypassed the tier gate on replay | 5384cfc | crafted tier=2 spool line searched the forge (`searched == []` failed with `['[auto-failure:s:E]']`) |
|
||||
| 6 | reconcile thread + worker race → duplicate issues | ba99269 (+10e9788 test correction) | race test produced 2 created issues (`assert len(created) == 1` failed with 2) |
|
||||
| 7 | unbounded queue.Queue | 03f606e | `maxsize == 0` assertion failed |
|
||||
| 8 | vacuous `forge_created_not_called()` (unconditional True) | 1c16d3d | test rewritten to real fake-state assertions |
|
||||
| 9 | unbounded error_class (10KB marker) | 1c16d3d | cap test failed (len 10000 ≠ 64) |
|
||||
| 10 | `filer: Any \| None` → protocol type | 8f5c161 (+01b6eb4 operator annotation) | AttributeError: `AutoFiler` missing (test asserts `get_type_hints` equality) |
|
||||
| 11 | plan's "maintenance cadence" reconcile call absent | 01b6eb4 (deviation recorded below) | n/a — verification + documentation |
|
||||
| 12 | O(N²) mark_consumed | resolved by finding-2 sidecar (no rewrite, O(1) append) | covered by finding-2 red tests |
|
||||
|
||||
**Deviation 8 (cadence):** the plan's "maintenance cadence" reconcile pass has
|
||||
no dedicated timer. The operator STARTUP thread (rune_operator.py:1986,
|
||||
`rune-autofile-reconcile`, launched by `_wire_auto_filer` — called once per
|
||||
process from `get_operator`) IS the only cadence: reconcile runs once per
|
||||
process boot, so spooled failures replay on restart. Accepted as deliberate
|
||||
scope: the filer spools on every failed write, the queue coalesces in-process
|
||||
retries, and a per-boot replay bounds the worst staleness to one restart; a
|
||||
recurring timer would add a background scheduler to a codebase that has none.
|
||||
Recorded per the deviations contract; no code added (the thread-launch comment
|
||||
names it the maintenance cadence).
|
||||
|
||||
## Research trace
|
||||
|
||||
Research round: issue #122 comment thread (posted 2026-09-07, amended after controller challenge — the amendment itself found the pagination defect).
|
||||
- subsystem_health.py:36 + forge_client.py:18-96 (INWARD) → composition-over-modification design; filer wraps, registry untouched.
|
||||
- ForgeClient.list_open_issues limit=50 no-pagination defect (forge_client.py:86-93, verified) → dedup via NEW search_issues protocol method (server-side, pagination-safe).
|
||||
- Event journal exists (memory/journal/events.jsonl) → spool rides existing durable store, no new subsystem.
|
||||
Kill test: if a T2 failure ever produces a client call, or a forge-down failure raises into the failure path, the design is wrong — frozen tests assert both.
|
||||
@@ -1,31 +0,0 @@
|
||||
# Research-gate evidence enforcement (pipeline hardening) — Plan
|
||||
|
||||
> **Contract:** controller-authored (controller-sized change), independent review gate + protected CI still apply. Conventional commit referencing the tranche epic #147 (§3.4-adjacent gate integrity). No deviation without plan-tail record.
|
||||
|
||||
## Goal
|
||||
|
||||
Nick's directive: research rounds must actually PULL sources so downstream stages work on evidence. Today `research-clear` clears on four regex keywords — a round of pure improvisation clears identically to a genuine one (observed this session). The gate must verify EVIDENCE per phase.
|
||||
|
||||
## Change (single job edit: `.gitea/workflows/pipeline-gate.yml` research-clear)
|
||||
|
||||
Split the comment body into phase sections (on the phase headers). Require:
|
||||
- **INWARD:** ≥2 codebase citations matching `(src|tests|scripts|docs|.gitea)/...` path pattern (evidence the code was actually read).
|
||||
- **OUTWARD:** ≥2 UNIQUE external http(s) URLs, excluding the forge's own host (self-references are not external sources) — evidence sources were actually pulled.
|
||||
- **INNOVATION + SYNTHESIS:** ≥1 total URL reference OR a `plans/` path mention (the synthesis must tie back to sources or a concrete plan artifact).
|
||||
- Missing evidence → the job posts a comment naming exactly what's missing (per phase), research-pending STAYS, no label transition. Existing success path unchanged otherwise.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- No LLM judging in CI (anti-complexity; regex evidence floor only).
|
||||
- No change to gate/teach jobs; no new labels.
|
||||
|
||||
## Tasks
|
||||
|
||||
1. Edit research-clear script: section splitter + evidence counters + rejection comment. Verify YAML parses; henxels exit 0; all canaries green (unchanged src → suite untouched but CI will run it on PR anyway).
|
||||
2. Commit `feat(pipeline): research gate requires evidence (sources in OUTWARD, citations in INWARD) (#147)`; push; PR; protected merge.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- A sourceless OUTWARD no longer clears the gate (rejection comment received).
|
||||
- An evidenced round (URLs + citations) clears as before.
|
||||
- Verified live: the next tranche research round (posted with general-3's pulled sources) passes the new gate.
|
||||
@@ -1,122 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""merge_robot.py — deterministic merge-on-green for this repo (#203).
|
||||
|
||||
Extracted from the inline workflow script so its logic is TESTABLE
|
||||
(bug-fix contract, docs/bugfix-contract.md: the silent no-op defect).
|
||||
|
||||
Behavior:
|
||||
- GET /user first: verify the token's identity and log it. Scoped-token
|
||||
failures are diagnosable from the job log instead of mysterious.
|
||||
- Scan open PRs; for each: combined status must be success AND mergeable.
|
||||
- Merge refusals (403/405/anything non-(200,405-already-merged)) FAIL THE JOB
|
||||
with the full error body — never print-and-continue. A robot that cannot
|
||||
merge must be visibly broken, not silently idle.
|
||||
- 405 with "behind the base branch" is retried once after a self-update of
|
||||
the head (informational), then fails loudly if still refused.
|
||||
- Every scanned PR produces a decision line: merged / skipped(reason).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
class MergeRefused(Exception):
|
||||
"""A merge the robot expected to perform was refused by the forge."""
|
||||
|
||||
|
||||
def _api(
|
||||
base: str, token: str, path: str, method: str = "GET", data: Any = None, ok: tuple = (200,)
|
||||
) -> tuple[int, Any]:
|
||||
req = urllib.request.Request(
|
||||
base + path,
|
||||
data=json.dumps(data).encode() if data is not None else None,
|
||||
headers={"Authorization": "token " + token, "Content-Type": "application/json"},
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
body = r.read()
|
||||
status = r.status
|
||||
except urllib.error.HTTPError as e:
|
||||
status = e.code
|
||||
body = e.read()
|
||||
except urllib.error.URLError as e:
|
||||
raise MergeRefused(f"forge unreachable at {path}: {e.reason}") from e
|
||||
parsed: Any = json.loads(body) if body else {}
|
||||
if method == "GET" and status not in ok:
|
||||
raise MergeRefused(f"GET {path} -> {status}: {str(parsed)[:300]}")
|
||||
return status, parsed
|
||||
|
||||
|
||||
def verify_identity(base: str, token: str) -> str:
|
||||
status, user = _api(base, token, "/user")
|
||||
login = user.get("login", "?")
|
||||
print(f"merge-robot: authenticated as {login} (status {status})")
|
||||
return login
|
||||
|
||||
|
||||
def scan(base: str, token: str) -> list[dict]:
|
||||
status, prs = _api(base, token, "/pulls?state=open&limit=50")
|
||||
print(f"merge-robot: {len(prs)} open PR(s)")
|
||||
return prs
|
||||
|
||||
|
||||
def decide(base: str, token: str, pr: dict) -> str:
|
||||
"""Return a decision line for one PR; raises MergeRefused on unexpected refusals."""
|
||||
n = pr["number"]
|
||||
sha = pr["head"]["sha"]
|
||||
status, cs = _api(base, token, f"/commits/{sha}/status")
|
||||
state = cs.get("state")
|
||||
if state != "success":
|
||||
return f"#{n}: skipped (status={state})"
|
||||
status, detail = _api(base, token, f"/pulls/{n}")
|
||||
if not detail.get("mergeable", False):
|
||||
return f"#{n}: skipped (not mergeable)"
|
||||
mstatus, mresp = _api(base, token, f"/pulls/{n}/merge", "POST", {"Do": "merge"})
|
||||
if mstatus == 200:
|
||||
return f"#{n}: MERGED"
|
||||
if mstatus == 405:
|
||||
msg = str(mresp.get("message", ""))
|
||||
if "behind" in msg.lower():
|
||||
return f"#{n}: skipped (head behind base — needs branch update): {msg}"
|
||||
raise MergeRefused(f"#{n} merge refused 405: {msg}")
|
||||
if mstatus == 403:
|
||||
raise MergeRefused(f"#{n} merge FORBIDDEN 403 — token lacks pull-request write scope: {str(mresp)[:300]}")
|
||||
raise MergeRefused(f"#{n} merge failed {mstatus}: {str(mresp)[:300]}")
|
||||
|
||||
|
||||
def main_with(argv: list[str]) -> int:
|
||||
token = argv[1] if len(argv) > 1 else ""
|
||||
base = argv[2] if len(argv) > 2 else ""
|
||||
if not token or not base:
|
||||
print("usage: merge_robot.py <token> <api-base>")
|
||||
return 2
|
||||
try:
|
||||
verify_identity(base, token)
|
||||
prs = scan(base, token)
|
||||
merged = 0
|
||||
for pr in prs:
|
||||
line = decide(base, token, pr)
|
||||
print("merge-robot:", line)
|
||||
if "MERGED" in line:
|
||||
merged += 1
|
||||
time.sleep(1)
|
||||
print(f"merge-robot: done — {merged} merged of {len(prs)} scanned")
|
||||
return 0
|
||||
except MergeRefused as exc:
|
||||
print(f"merge-robot: FAILING JOB — {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
return main_with(sys.argv)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -3,4 +3,4 @@
|
||||
This is the main entry point for the Rune program.
|
||||
"""
|
||||
|
||||
__version__ = "0.54.0"
|
||||
__version__ = "0.50.0"
|
||||
@@ -15,7 +15,6 @@ import json
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import warnings
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, List
|
||||
|
||||
@@ -90,32 +89,8 @@ class AttentionCapsule:
|
||||
messages.append({"role": "user", "content": user_content})
|
||||
return messages
|
||||
|
||||
def capture_messages(self, messages: List[dict]) -> List[dict]:
|
||||
"""Record the EXACT message list sent to the runtime (P1-02 #25).
|
||||
|
||||
Stored post-hoc on ``capsule.sent_messages`` (dynamic attr, same
|
||||
pattern as ``messages``/``system_preamble`` from the broker render)
|
||||
so the equality invariant — ``to_messages()`` re-derivation ==
|
||||
what the model received — is machine-checkable from the ledger.
|
||||
The list is deep-copied so later caller-side mutation can never
|
||||
diverge the logged payload from what was sent.
|
||||
"""
|
||||
self.sent_messages = json.loads(json.dumps(list(messages)))
|
||||
return self.sent_messages
|
||||
|
||||
def to_prompt(self) -> str:
|
||||
"""Convert capsule to prompt text.
|
||||
|
||||
.. deprecated:: P1-02 (#25)
|
||||
Legacy renderer behind the retired string-trim "budgeting" path.
|
||||
The ContextBroker owns prompt assembly; use ``to_messages()``.
|
||||
"""
|
||||
warnings.warn(
|
||||
"attention_capsule.AttentionCapsule.to_prompt is deprecated (P1-02 #25): "
|
||||
"the ContextBroker owns prompt assembly — use to_messages().",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
"""Convert capsule to prompt text."""
|
||||
parts = []
|
||||
|
||||
if self.active_objective:
|
||||
@@ -160,21 +135,9 @@ def estimate_tokens(text: str) -> int:
|
||||
|
||||
def create_capsule(task: str, context: str = "", max_tokens: int = 600) -> AttentionCapsule:
|
||||
"""Create an attention capsule for a task.
|
||||
|
||||
|
||||
Only includes what's relevant to the current task.
|
||||
|
||||
.. deprecated:: P1-02 (#25)
|
||||
Legacy CLI helper with fake string-trim "budgeting" — token counts
|
||||
here are never the real admission decision. The ContextBroker
|
||||
(#19) is the sole prompt-assembly authority. Kept only for the
|
||||
standalone CLI above; no live callers.
|
||||
"""
|
||||
warnings.warn(
|
||||
"attention_capsule.create_capsule is deprecated (P1-02 #25): its string-trim "
|
||||
"budgeting is not real admission — use the ContextBroker.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
capsule = AttentionCapsule()
|
||||
|
||||
# Extract objective from task
|
||||
|
||||
@@ -252,16 +252,11 @@ def main() -> int:
|
||||
)
|
||||
check(
|
||||
"Rune minor version advertises provisioning and restore verification",
|
||||
'__version__ = "0.54.0"' in rune_version,
|
||||
'__version__ = "0.50.0"' in rune_version,
|
||||
failures,
|
||||
)
|
||||
# #169 policy update: auto-bump on merge is sanctioned (Nick directive:
|
||||
# bumping is automatic from the Gitea end). The push must be an explicit,
|
||||
# visible auto-bump commit — not a hidden rewrite.
|
||||
check(
|
||||
"release automation push is the sanctioned auto-bump only",
|
||||
"git push" not in versioning or "auto-bump version to" in versioning,
|
||||
failures,
|
||||
"release automation cannot push master", "git push" not in versioning, failures
|
||||
)
|
||||
check(
|
||||
"release API URL comes from Gitea context",
|
||||
|
||||
@@ -1,279 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""P1-01 capability-authority canary (#24).
|
||||
|
||||
Self-bootstrapping like canary_p1_06.py (CI runs ``python3 <canary>``
|
||||
without PYTHONPATH). Proves the #24 claims are machine-checked with NO
|
||||
live model, NO Ollama, NO network:
|
||||
|
||||
1. bounded exposure: a coding job's broker selection exposes ~5–10
|
||||
schemas (never all 19, never empty, floor + escape hatch present);
|
||||
2. zero-token claim: the selected tools payload is byte-smaller than the
|
||||
full registry payload (unselected capabilities cost nothing);
|
||||
3. escape-hatch round-trip: a scripted LocalToolRuntime run where the
|
||||
model calls find_capability and then calls the discovered tool in the
|
||||
SAME run (dynamic exposure within the iteration budget);
|
||||
4. floor guarantee: a zero-score task still yields the five T1 file
|
||||
tools + find_capability with reason "floor (no matches)";
|
||||
5. scripted operator run: the real RuneOperator feeds the selected set
|
||||
into the runtime construction site and persists trace-v3 capability
|
||||
fields (selected_tools / capability_reason / find_capability_calls).
|
||||
|
||||
Isolates all state under RUNE_HOME so the canary never touches a real
|
||||
home directory. Prints 'P1-01 CAPABILITY AUTHORITY CANARY PASSED' and
|
||||
exits 0.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
# Self-bootstrap: CI runs `python3 <canary>` without PYTHONPATH, so put the
|
||||
# repository src/ on sys.path before importing rune modules. Isolate Rune's
|
||||
# state/journal paths so the canary never touches a real home directory.
|
||||
_repo_src = str(Path(__file__).resolve().parents[1])
|
||||
sys.path.insert(0, _repo_src)
|
||||
_canary_home = tempfile.mkdtemp(prefix="p1-01-canary-home-")
|
||||
os.environ["RUNE_HOME"] = _canary_home
|
||||
os.environ["RUNE_MEMORY_DIR"] = tempfile.mkdtemp(prefix="p1-01-canary-mem-")
|
||||
|
||||
from rune import capability_broker as cb # noqa: E402
|
||||
from rune import inference_trace as it # noqa: E402
|
||||
from rune import rune_operator # noqa: E402
|
||||
from rune.capability_broker import select_for_job # noqa: E402
|
||||
from rune.local_tool_runtime import LocalToolRuntime # noqa: E402
|
||||
from rune.tool_executor import TOOLS, execute_tool # noqa: E402
|
||||
from rune.tool_protocol import to_ollama_tools # noqa: E402
|
||||
|
||||
FLOOR = set(cb.FLOOR_TOOLS)
|
||||
HATCH = cb.ESCAPE_HATCH_TOOL
|
||||
|
||||
|
||||
def _fail(message: str) -> int:
|
||||
print(f"FAILED: {message}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def check_bounded_exposure() -> int:
|
||||
"""A coding job exposes ~5–10 schemas, never the full registry."""
|
||||
selection = select_for_job("fix the failing test in the parser module and refactor it")
|
||||
names = selection["names"]
|
||||
if not (5 <= len(names) <= 10):
|
||||
return _fail(f"selection size {len(names)} outside 5..10: {names}")
|
||||
if len(names) >= len(TOOLS):
|
||||
return _fail(f"selection ({len(names)}) did not shrink below the registry ({len(TOOLS)})")
|
||||
if not FLOOR <= set(names):
|
||||
return _fail(f"floor missing from selection: {names}")
|
||||
if HATCH not in names:
|
||||
return _fail("escape hatch not exposed")
|
||||
if selection["reason"] == "":
|
||||
return _fail("selection reason not recorded")
|
||||
if not selection["task_family"]:
|
||||
return _fail("task family not recorded")
|
||||
return 0
|
||||
|
||||
|
||||
def check_zero_token_payload() -> int:
|
||||
"""Unselected capabilities cost zero bytes in the tools payload."""
|
||||
selection = select_for_job("fix the bug in parse.py")
|
||||
selected_payload = json.dumps(to_ollama_tools(selection["tools"]))
|
||||
full_payload = json.dumps(to_ollama_tools(TOOLS))
|
||||
if len(selected_payload) >= len(full_payload):
|
||||
return _fail(f"selected payload ({len(selected_payload)}B) is not smaller than full ({len(full_payload)}B)")
|
||||
exposed = {item["function"]["name"] for item in to_ollama_tools(selection["tools"])}
|
||||
if exposed != set(selection["names"]):
|
||||
return _fail("payload names diverge from the selection")
|
||||
return 0
|
||||
|
||||
|
||||
def check_floor_guarantee() -> int:
|
||||
"""Zero-score task → floor + hatch, reason 'floor (no matches)'."""
|
||||
selection = select_for_job("zzzqqq xxxyyy wwww vvvv")
|
||||
names = set(selection["names"])
|
||||
if names != FLOOR | {HATCH}:
|
||||
return _fail(f"zero-score selection must be floor+hatch, got {sorted(names)}")
|
||||
if selection["reason"] != "floor (no matches)":
|
||||
return _fail(f"floor reason wrong: {selection['reason']!r}")
|
||||
return 0
|
||||
|
||||
|
||||
class _FakeTransport:
|
||||
"""Scripted native-tools transport capturing every tools payload."""
|
||||
|
||||
def __init__(self, responses):
|
||||
self.responses = list(responses)
|
||||
self.calls = []
|
||||
|
||||
def supports_tools(self, model: str) -> bool:
|
||||
return True
|
||||
|
||||
def supports_thinking(self, model: str) -> bool:
|
||||
return True
|
||||
|
||||
def chat(self, *, model, messages, tools, options=None, think=None):
|
||||
self.calls.append({"tools": tools, "messages": messages})
|
||||
response = self.responses.pop(0)
|
||||
if isinstance(response, BaseException):
|
||||
raise response
|
||||
return response
|
||||
|
||||
|
||||
def _native_call(call_id, name, arguments):
|
||||
return {"id": call_id, "function": {"name": name, "arguments": arguments}}
|
||||
|
||||
|
||||
def check_escape_hatch_round_trip() -> int:
|
||||
"""find_capability → discovered tool callable in the SAME run."""
|
||||
selection = select_for_job("fix the bug in parse.py")
|
||||
if "repo_map" in selection["names"]:
|
||||
return _fail("repo_map unexpectedly pre-exposed; pick another discovery target")
|
||||
transport = _FakeTransport(
|
||||
[
|
||||
{
|
||||
"content": "",
|
||||
"tool_calls": [_native_call("c1", HATCH, {"query": "map the repository structure"})],
|
||||
},
|
||||
{
|
||||
"content": "",
|
||||
"tool_calls": [_native_call("c2", "repo_map", {"path": "."})],
|
||||
},
|
||||
{"content": "Repository mapped."},
|
||||
]
|
||||
)
|
||||
runtime = LocalToolRuntime(
|
||||
transport=transport,
|
||||
executor=execute_tool,
|
||||
model="canary-model",
|
||||
tools=selection["tools"],
|
||||
)
|
||||
result = runtime.run([{"role": "user", "content": "map the repo"}])
|
||||
if not result.success:
|
||||
return _fail(f"escape-hatch round-trip run failed: {result.error}")
|
||||
if result.user_text != "Repository mapped.":
|
||||
return _fail(f"run did not reach the final turn: {result.user_text!r}")
|
||||
second_names = {item["function"]["name"] for item in transport.calls[1]["tools"]}
|
||||
if "repo_map" not in second_names:
|
||||
return _fail("discovered tool was not exposed within the same run")
|
||||
if runtime.find_capability_calls != 1:
|
||||
return _fail(f"find_capability_calls must be 1, got {runtime.find_capability_calls}")
|
||||
# The hatch result itself must carry the schema summary.
|
||||
hatch_event = next((e for e in result.tool_events or [] if e.request.name == HATCH), None)
|
||||
if hatch_event is None:
|
||||
return _fail("no find_capability tool event in the run")
|
||||
hatch_result = hatch_event.result or {}
|
||||
if "repo_map" not in str(hatch_result.get("result", "")):
|
||||
return _fail("hatch result missing the discovered capability summary")
|
||||
return 0
|
||||
|
||||
|
||||
def check_scripted_operator_wiring() -> int:
|
||||
"""Real RuneOperator feeds the selection to the runtime + trace v3."""
|
||||
storage = Path(_canary_home) / "operator.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
trace = {
|
||||
"model_id": "qwen3:14b",
|
||||
"window": 40960,
|
||||
"budget": 32768,
|
||||
"blocks": [
|
||||
{"name": "persona", "tokens": 100, "admitted": True, "reason": ""},
|
||||
{"name": "user", "tokens": 20, "admitted": True, "reason": ""},
|
||||
],
|
||||
"admitted_total": 120,
|
||||
"dropped": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"compaction": [],
|
||||
"elapsed_ms": 1.0,
|
||||
}
|
||||
capsule = SimpleNamespace(
|
||||
token_count=120,
|
||||
active_objective="",
|
||||
blocks=[],
|
||||
to_messages=lambda: [{"role": "user", "content": "do it"}],
|
||||
)
|
||||
broker = SimpleNamespace(assemble=lambda task, session, model_id: (capsule, trace))
|
||||
response = rune_operator.OperatorResponse(success=True, user_text="done", model="qwen3:14b")
|
||||
captured: dict = {}
|
||||
|
||||
class _ScriptedRuntime:
|
||||
transport = Mock(provider="ollama")
|
||||
tool_names = frozenset()
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
|
||||
def run(self, messages, **kwargs):
|
||||
return response
|
||||
|
||||
patches = [
|
||||
patch.object(op, "_context_broker", return_value=broker),
|
||||
patch.object(
|
||||
rune_operator.model_profiles, "get_or_create_profile", return_value=SimpleNamespace(safety_tier=0)
|
||||
),
|
||||
patch.object(rune_operator.model_profiles, "record_interaction"),
|
||||
patch.object(rune_operator, "LocalToolRuntime", _ScriptedRuntime),
|
||||
patch.object(rune_operator, "OllamaChatTransport", return_value=Mock(provider="ollama")),
|
||||
patch("rune.local_tool_runtime.context_length", lambda model: 40960),
|
||||
patch.object(
|
||||
rune_operator.safety_override.SafetyOverride,
|
||||
"get_params",
|
||||
lambda self, text: {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""},
|
||||
),
|
||||
patch.object(rune_operator.safety_override.SafetyOverride, "is_refusal", lambda self, text: False),
|
||||
patch.object(op, "_log_token_usage"),
|
||||
]
|
||||
with contextlib.ExitStack() as stack:
|
||||
for item in patches:
|
||||
stack.enter_context(item)
|
||||
stack.enter_context(patch("rune.event_bus._store_path", lambda: Path(storage).parent / "journal-events.jsonl"))
|
||||
result = op.send_task("fix this code path")
|
||||
|
||||
if not result.get("success"):
|
||||
return _fail("scripted operator run did not succeed")
|
||||
runtime_tools = captured.get("tools")
|
||||
if runtime_tools is None:
|
||||
return _fail("operator did not pass tools= to the runtime construction site")
|
||||
expected = select_for_job("fix this code path")["names"]
|
||||
if [t["name"] for t in runtime_tools] != expected:
|
||||
return _fail(f"runtime received {sorted(t['name'] for t in runtime_tools)}, expected {expected}")
|
||||
if not (5 <= len(expected) <= 10):
|
||||
return _fail(f"operator-exposed set size {len(expected)} outside 5..10")
|
||||
|
||||
records = it.read_traces(trace_dir=Path(_canary_home) / "state")
|
||||
if not records:
|
||||
return _fail("no durable inference trace persisted")
|
||||
record = records[-1]
|
||||
if record["schema_version"] != 4:
|
||||
return _fail(f"trace schema_version must be 4, got {record['schema_version']}")
|
||||
if record["selected_tools"] != expected:
|
||||
return _fail(f"trace selected_tools wrong: {record['selected_tools']}")
|
||||
if not record["capability_reason"]:
|
||||
return _fail("trace capability_reason empty")
|
||||
if record["find_capability_calls"] != 0:
|
||||
return _fail(f"find_capability_calls must be 0, got {record['find_capability_calls']}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
checks = (
|
||||
("bounded exposure: coding job sees ~5-10 schemas, floor + hatch present", check_bounded_exposure),
|
||||
("zero-token claim: selected tools payload is byte-smaller than full", check_zero_token_payload),
|
||||
("escape hatch: find_capability then discovered tool in the SAME run", check_escape_hatch_round_trip),
|
||||
("floor guarantee: zero-score task still yields floor + hatch", check_floor_guarantee),
|
||||
("operator wiring: selection reaches the runtime + trace v3 fields", check_scripted_operator_wiring),
|
||||
)
|
||||
for label, check in checks:
|
||||
result = check()
|
||||
if result != 0:
|
||||
return result
|
||||
print(f" ✓ {label}")
|
||||
print("P1-01 CAPABILITY AUTHORITY CANARY PASSED")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,253 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""P1-02 capsule-fidelity canary (#25).
|
||||
|
||||
Self-bootstrapping like canary_p1_07.py (CI runs ``python3 <canary>``
|
||||
without PYTHONPATH). Proves the #25 claims are machine-checked with NO
|
||||
live model, NO Ollama, NO network:
|
||||
|
||||
1. trace.capsule_messages == the messages the runtime received (exact
|
||||
list identity) — the acceptance invariant;
|
||||
2. capsule_token_count is truthful post-safety-override decoration;
|
||||
3. num_ctx == capsule.token_count + completion_reserve (the broker
|
||||
budget basis), NOT the raw window;
|
||||
4. content hashes match the block contents;
|
||||
5. no DeprecationWarning is suppressed: the legacy create_capsule /
|
||||
to_prompt paths still warn.
|
||||
|
||||
Isolates all state under RUNE_HOME so the canary never touches a real
|
||||
home directory. Prints 'P1-02 CAPSULE FIDELITY CANARY PASSED' and exits 0.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
# Self-bootstrap: CI runs `python3 <canary>` without PYTHONPATH, so put the
|
||||
# repository src/ on sys.path before importing rune modules. Isolate Rune's
|
||||
# state/journal paths so the canary never touches a real home directory.
|
||||
_repo_src = str(Path(__file__).resolve().parents[1])
|
||||
sys.path.insert(0, _repo_src)
|
||||
_canary_home = tempfile.mkdtemp(prefix="p1-02-canary-home-")
|
||||
os.environ["RUNE_HOME"] = _canary_home
|
||||
os.environ["RUNE_MEMORY_DIR"] = tempfile.mkdtemp(prefix="p1-02-canary-mem-")
|
||||
|
||||
from rune import rune_operator # noqa: E402
|
||||
from rune.attention_capsule import AttentionCapsule # noqa: E402
|
||||
from rune.context_broker import ContributionBlock # noqa: E402
|
||||
from rune.inference_trace import read_traces # noqa: E402
|
||||
from rune.local_tool_runtime import OllamaChatTransport # noqa: E402
|
||||
from rune.tool_protocol import OperatorResponse # noqa: E402
|
||||
|
||||
WINDOW = 40960
|
||||
BUDGET = 32768
|
||||
|
||||
|
||||
def _fail(message: str) -> int:
|
||||
print(f"FAILED: {message}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
class _ScriptedTransport(OllamaChatTransport):
|
||||
"""OllamaChatTransport with _post scripted — real accumulation code runs."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def _post(self, path, payload):
|
||||
return {
|
||||
"model": "qwen3:8b",
|
||||
"created_at": "2026-09-05T00:00:00Z",
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"done": True,
|
||||
"prompt_eval_count": 100,
|
||||
"eval_count": 5,
|
||||
"total_duration": 1_000_000_000,
|
||||
}
|
||||
|
||||
|
||||
class _ScriptedRuntime:
|
||||
"""Minimal runtime stand-in: one chat, then the canned final response."""
|
||||
|
||||
def __init__(self, transport):
|
||||
self.transport = transport
|
||||
self.tool_names = ("read_file", "write_file")
|
||||
self.received_messages: list = []
|
||||
self.received_options: dict = {}
|
||||
|
||||
def run(self, messages, **kwargs):
|
||||
self.received_messages = [dict(m) for m in messages]
|
||||
self.received_options = dict(kwargs.get("options") or {})
|
||||
self.transport.chat(model="qwen3:8b", messages=list(messages), tools=None)
|
||||
return OperatorResponse(success=True, user_text="All done.", model="qwen3:8b")
|
||||
|
||||
|
||||
def _scripted_operator_run(description: str, tier_params: dict):
|
||||
"""Real RuneOperator + scripted broker/transport → (envelope, runtime, capsule)."""
|
||||
storage = Path(_canary_home) / "operator.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
capsule = AttentionCapsule()
|
||||
capsule.blocks = [
|
||||
ContributionBlock("persona", 90, "I am Rune.", len("I am Rune.") // 4, "persona", evictable=False),
|
||||
ContributionBlock("user", 100, description, len(description) // 4, "rune_operator task text", evictable=False),
|
||||
]
|
||||
capsule.token_count = sum(b.tokens for b in capsule.blocks)
|
||||
trace = {
|
||||
"model_id": "qwen3:8b",
|
||||
"window": WINDOW,
|
||||
"budget": BUDGET,
|
||||
"blocks": [
|
||||
{"name": "persona", "tokens": capsule.blocks[0].tokens, "admitted": True, "reason": ""},
|
||||
{"name": "user", "tokens": capsule.blocks[1].tokens, "admitted": True, "reason": ""},
|
||||
],
|
||||
"admitted": ["persona", "user"],
|
||||
"admitted_total": capsule.token_count,
|
||||
"dropped": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"compaction": [],
|
||||
"elapsed_ms": 1.0,
|
||||
}
|
||||
broker = SimpleNamespace(assemble=lambda task, session, model_id: (capsule, trace))
|
||||
runtime = _ScriptedRuntime(_ScriptedTransport())
|
||||
profile = SimpleNamespace(safety_tier=0)
|
||||
patches = [
|
||||
patch.object(op, "_context_broker", return_value=broker),
|
||||
patch.object(rune_operator.model_profiles, "get_or_create_profile", return_value=profile),
|
||||
patch.object(rune_operator.model_profiles, "record_interaction"),
|
||||
patch.object(rune_operator, "LocalToolRuntime", return_value=runtime),
|
||||
patch.object(rune_operator, "OllamaChatTransport", return_value=runtime.transport),
|
||||
patch("rune.local_tool_runtime.context_length", lambda model: WINDOW),
|
||||
patch.object(rune_operator.safety_override.SafetyOverride, "get_params", lambda self, text: dict(tier_params)),
|
||||
patch.object(rune_operator.safety_override.SafetyOverride, "is_refusal", lambda self, text: False),
|
||||
patch.object(op, "_log_token_usage"),
|
||||
]
|
||||
with contextlib.ExitStack() as stack:
|
||||
for item in patches:
|
||||
stack.enter_context(item)
|
||||
stack.enter_context(patch("rune.event_bus._store_path", lambda: Path(storage).parent / "journal-events.jsonl"))
|
||||
envelope = op.send_task(description, "qwen3:8b")
|
||||
return envelope, runtime, capsule
|
||||
|
||||
|
||||
def check_message_identity() -> int:
|
||||
"""Invariant: logged capsule messages == runtime-received messages."""
|
||||
description = "inspect this code path"
|
||||
envelope, runtime, capsule = _scripted_operator_run(
|
||||
description, {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""}
|
||||
)
|
||||
if not envelope.get("success"):
|
||||
return _fail(f"scripted run failed: {envelope.get('error')}")
|
||||
records = read_traces(trace_dir=Path(_canary_home) / "state")
|
||||
if not records:
|
||||
return _fail("no durable inference trace persisted")
|
||||
record = records[-1]
|
||||
if record["capsule_messages"] != runtime.received_messages:
|
||||
return _fail("trace.capsule_messages != runtime-received messages (acceptance invariant)")
|
||||
if record["capsule_messages"][-1]["content"] != description:
|
||||
return _fail("logged user content diverges from the task description")
|
||||
if capsule.sent_messages != capsule.to_messages():
|
||||
return _fail("captured sent messages != to_messages() re-derivation")
|
||||
return 0
|
||||
|
||||
|
||||
def check_token_truthfulness() -> int:
|
||||
"""Post-decoration token_count is re-measured (not stale)."""
|
||||
description = "please inspect the failing test"
|
||||
suffix = "\n\nI respond as myself:"
|
||||
envelope, _runtime, capsule = _scripted_operator_run(
|
||||
description, {"tier": 2, "temperature": 0.95, "prefix": "", "suffix": suffix}
|
||||
)
|
||||
if not envelope.get("success"):
|
||||
return _fail(f"scripted run failed: {envelope.get('error')}")
|
||||
user_block = next(b for b in capsule.blocks if b.name == "user")
|
||||
if user_block.content != description + suffix:
|
||||
return _fail("safety override did not decorate the user block")
|
||||
expected = sum(len(b.content) // 4 if isinstance(b.content, str) else 0 for b in capsule.blocks)
|
||||
if capsule.token_count != expected:
|
||||
return _fail(f"capsule.token_count {capsule.token_count} != re-measured {expected}")
|
||||
records = read_traces(trace_dir=Path(_canary_home) / "state")
|
||||
if records[-1]["capsule_token_count"] != expected:
|
||||
return _fail("trace capsule_token_count not truthful post-decoration")
|
||||
return 0
|
||||
|
||||
|
||||
def check_num_ctx_budget() -> int:
|
||||
"""num_ctx == capsule.token_count + completion_reserve, never the window."""
|
||||
description = "inspect this code path"
|
||||
envelope, runtime, capsule = _scripted_operator_run(
|
||||
description, {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""}
|
||||
)
|
||||
if not envelope.get("success"):
|
||||
return _fail(f"scripted run failed: {envelope.get('error')}")
|
||||
completion_reserve = WINDOW - BUDGET
|
||||
expected = capsule.token_count + completion_reserve
|
||||
if runtime.received_options.get("num_ctx") != expected:
|
||||
return _fail(f"num_ctx {runtime.received_options.get('num_ctx')} != budget basis {expected}")
|
||||
if runtime.received_options.get("num_ctx") == WINDOW:
|
||||
return _fail("num_ctx equals the raw window — budget not respected")
|
||||
return 0
|
||||
|
||||
|
||||
def check_content_hashes() -> int:
|
||||
"""Per-block SHA-256 hashes match the actual block contents."""
|
||||
description = "hash integrity probe"
|
||||
envelope, _runtime, capsule = _scripted_operator_run(
|
||||
description, {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""}
|
||||
)
|
||||
if not envelope.get("success"):
|
||||
return _fail(f"scripted run failed: {envelope.get('error')}")
|
||||
records = read_traces(trace_dir=Path(_canary_home) / "state")
|
||||
hashes = records[-1]["capsule_content_hashes"]
|
||||
if set(hashes) != {b.name for b in capsule.blocks}:
|
||||
return _fail(f"hash keys {sorted(hashes)} != block names {sorted(b.name for b in capsule.blocks)}")
|
||||
for block in capsule.blocks:
|
||||
payload = (
|
||||
block.content.encode("utf-8")
|
||||
if isinstance(block.content, str)
|
||||
else json.dumps(block.content, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
)
|
||||
if hashes[block.name] != hashlib.sha256(payload).hexdigest():
|
||||
return _fail(f"content hash mismatch for block '{block.name}'")
|
||||
return 0
|
||||
|
||||
|
||||
def check_deprecation_not_suppressed() -> int:
|
||||
"""Legacy paths still warn — no silent suppression of the retirement."""
|
||||
from rune.attention_capsule import create_capsule
|
||||
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
create_capsule("fix the failing test", "context line here")
|
||||
AttentionCapsule(active_objective="obj").to_prompt()
|
||||
deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)]
|
||||
if len(deprecations) < 2:
|
||||
return _fail(f"expected 2 DeprecationWarnings from legacy paths, got {len(deprecations)}")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
checks = (
|
||||
("trace.capsule_messages == runtime-received messages (identity)", check_message_identity),
|
||||
("capsule_token_count truthful post-decoration", check_token_truthfulness),
|
||||
("num_ctx == budget basis, not the raw window", check_num_ctx_budget),
|
||||
("content hashes match block contents", check_content_hashes),
|
||||
("legacy deprecation warnings not suppressed", check_deprecation_not_suppressed),
|
||||
)
|
||||
for label, check in checks:
|
||||
result = check()
|
||||
if result != 0:
|
||||
return result
|
||||
print(f" ✓ {label}")
|
||||
print("P1-02 CAPSULE FIDELITY CANARY PASSED")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,267 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""P1-06 model/provider-routing canary (#29).
|
||||
|
||||
Self-bootstrapping like canary_p1_07.py (CI runs ``python3 <canary>``
|
||||
without PYTHONPATH). Proves the #29 claims are machine-checked with NO
|
||||
live model, NO Ollama, NO network:
|
||||
|
||||
1. rank order: the advisory ModelRouter ranks verified-capable candidates
|
||||
cheapest-first with measured evidence reasons and respects the
|
||||
capability floor (need_tools, context);
|
||||
2. reason recording: a scripted operator run (real RuneOperator, scripted
|
||||
router/broker/transport) persists a schema-v2 inference trace carrying
|
||||
routing_reason / routing_mode / task_family;
|
||||
3. provider-removal profile persistence: deleting the adapter object
|
||||
leaves the measured-profiles file on disk intact.
|
||||
|
||||
Isolates all state under RUNE_HOME so the canary never touches a real
|
||||
home directory. Prints 'P1-06 MODEL ROUTING CANARY PASSED' and exits 0.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
# Self-bootstrap: CI runs `python3 <canary>` without PYTHONPATH, so put the
|
||||
# repository src/ on sys.path before importing rune modules. Isolate Rune's
|
||||
# state/journal paths so the canary never touches a real home directory.
|
||||
_repo_src = str(Path(__file__).resolve().parents[1])
|
||||
sys.path.insert(0, _repo_src)
|
||||
_canary_home = tempfile.mkdtemp(prefix="p1-06-canary-home-")
|
||||
os.environ["RUNE_HOME"] = _canary_home
|
||||
os.environ["RUNE_MEMORY_DIR"] = tempfile.mkdtemp(prefix="p1-06-canary-mem-")
|
||||
|
||||
from rune import rune_operator # noqa: E402
|
||||
from rune.local_tool_runtime import OllamaChatTransport # noqa: E402
|
||||
from rune.model_routing import ( # noqa: E402
|
||||
ModelRouter,
|
||||
ProviderUnavailable,
|
||||
load_profiles,
|
||||
save_profiles,
|
||||
)
|
||||
from rune.tool_protocol import OperatorResponse # noqa: E402
|
||||
|
||||
|
||||
def _fail(message: str) -> int:
|
||||
print(f"FAILED: {message}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def check_rank_order() -> int:
|
||||
"""Cheapest verified-capable first; evidence reasons; capability floor."""
|
||||
path = Path(_canary_home) / "rank-profiles.json"
|
||||
save_profiles(
|
||||
path,
|
||||
{
|
||||
("ollama", "qwen3:14b"): SimpleNamespace(
|
||||
per_family_success={"code": [12, 13]},
|
||||
avg_latency_ms=0.0,
|
||||
estimated_cost_usd=0.0,
|
||||
),
|
||||
("openai", "mimo-v2.5-pro"): SimpleNamespace(
|
||||
per_family_success={"code": [9, 10]},
|
||||
avg_latency_ms=0.0,
|
||||
estimated_cost_usd=1.0,
|
||||
),
|
||||
},
|
||||
)
|
||||
router = ModelRouter(profiles_path=path)
|
||||
|
||||
# measured verification overrides the static tool claim, cost orders
|
||||
ranked = router.rank(task_family="code", need_tools=True)
|
||||
if not ranked:
|
||||
return _fail("rank() returned no verified-capable candidates")
|
||||
if ranked[0].model != "qwen3:14b" or ranked[0].estimated_cost != 0.0:
|
||||
return _fail("cheapest verified-capable model is not ranked first")
|
||||
if ranked[0].reason != "measured 12/13 success task_family=code":
|
||||
return _fail(f"measured evidence reason missing: {ranked[0].reason!r}")
|
||||
|
||||
# zero measured success in a family → not verified-capable
|
||||
empty = ModelRouter(
|
||||
candidates=(("bad-model", "ollama", "T1", True, 32768, 0.0),),
|
||||
profiles={
|
||||
("ollama", "bad-model"): SimpleNamespace(
|
||||
per_family_success={"code": [0, 5]},
|
||||
avg_latency_ms=0.0,
|
||||
estimated_cost_usd=0.0,
|
||||
)
|
||||
},
|
||||
)
|
||||
if empty.rank(task_family="code"):
|
||||
return _fail("zero-success model must not be verified-capable")
|
||||
|
||||
# capability floor without measured profiles
|
||||
floor = ModelRouter()
|
||||
ranked_floor = floor.rank(task_family="code", need_tools=True)
|
||||
if [d.model for d in ranked_floor] != ["mimo-v2.5-pro"]:
|
||||
return _fail("capability floor with need_tools must leave only T4")
|
||||
if not ranked_floor[0].reason.startswith("no-measured-profiles"):
|
||||
return _fail("floor reason must name no-measured-profiles")
|
||||
return 0
|
||||
|
||||
|
||||
class _ScriptedTransport(OllamaChatTransport):
|
||||
"""OllamaChatTransport with accumulators preset — no network."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.provider = "ollama"
|
||||
self.last_prompt_eval_count = None
|
||||
self.usage_calls = 1
|
||||
self.usage_prompt_tokens = 120
|
||||
self.usage_output_tokens = 10
|
||||
self.usage_total_duration_ns = 1_000_000
|
||||
|
||||
|
||||
def _scripted_operator_run():
|
||||
"""Real RuneOperator + scripted router/broker/transport → send_task."""
|
||||
storage = Path(_canary_home) / "operator.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
|
||||
decision = SimpleNamespace(
|
||||
model="qwen3:14b",
|
||||
provider="ollama",
|
||||
tier="T2",
|
||||
reason="measured 12/13 success task_family=code",
|
||||
estimated_cost=0.0,
|
||||
)
|
||||
scripted_router = SimpleNamespace(rank=lambda *a, **k: [decision])
|
||||
|
||||
trace = {
|
||||
"model_id": "qwen3:14b",
|
||||
"window": 40960,
|
||||
"budget": 32768,
|
||||
"blocks": [
|
||||
{"name": "persona", "tokens": 100, "admitted": True, "reason": ""},
|
||||
{"name": "user", "tokens": 20, "admitted": True, "reason": ""},
|
||||
],
|
||||
"admitted_total": 120,
|
||||
"dropped": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"compaction": [],
|
||||
"elapsed_ms": 1.0,
|
||||
}
|
||||
capsule = SimpleNamespace(
|
||||
token_count=120,
|
||||
active_objective="",
|
||||
blocks=[],
|
||||
to_messages=lambda: [{"role": "user", "content": "do it"}],
|
||||
)
|
||||
broker = SimpleNamespace(assemble=lambda task, session, model_id: (capsule, trace))
|
||||
profile = SimpleNamespace(safety_tier=0)
|
||||
response = OperatorResponse(success=True, user_text="done", model="qwen3:14b")
|
||||
|
||||
class _ScriptedRuntime:
|
||||
def __init__(self):
|
||||
self.transport = _ScriptedTransport()
|
||||
|
||||
def run(self, messages, **kwargs):
|
||||
return response
|
||||
|
||||
runtime = _ScriptedRuntime()
|
||||
patches = [
|
||||
patch.object(op, "_context_broker", return_value=broker),
|
||||
patch.object(rune_operator.model_profiles, "get_or_create_profile", return_value=profile),
|
||||
patch.object(rune_operator.model_profiles, "record_interaction"),
|
||||
patch.object(rune_operator, "LocalToolRuntime", return_value=runtime),
|
||||
patch.object(rune_operator, "OllamaChatTransport", return_value=runtime.transport),
|
||||
patch("rune.local_tool_runtime.context_length", lambda model: 40960),
|
||||
patch.object(
|
||||
rune_operator.safety_override.SafetyOverride,
|
||||
"get_params",
|
||||
lambda self, text: {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""},
|
||||
),
|
||||
patch.object(rune_operator.safety_override.SafetyOverride, "is_refusal", lambda self, text: False),
|
||||
patch.object(op, "_log_token_usage"),
|
||||
patch.object(rune_operator, "_build_model_router", return_value=scripted_router),
|
||||
]
|
||||
with contextlib.ExitStack() as stack:
|
||||
for item in patches:
|
||||
stack.enter_context(item)
|
||||
stack.enter_context(patch("rune.event_bus._store_path", lambda: Path(storage).parent / "journal-events.jsonl"))
|
||||
return op.send_task("fix this code path")
|
||||
|
||||
|
||||
def check_reason_recording() -> int:
|
||||
"""Schema-v2 trace carries routing_reason / routing_mode / task_family."""
|
||||
from rune import inference_trace as it
|
||||
|
||||
result = _scripted_operator_run()
|
||||
if not result.get("success"):
|
||||
return _fail("scripted router-driven run did not succeed")
|
||||
records = it.read_traces(trace_dir=Path(_canary_home) / "state")
|
||||
if not records:
|
||||
return _fail("no durable inference trace persisted")
|
||||
record = records[-1]
|
||||
if record["schema_version"] != 4:
|
||||
return _fail(f"schema_version must be 4, got {record['schema_version']}")
|
||||
if record["routing_reason"] != "router: measured 12/13 success task_family=code":
|
||||
return _fail(f"routing_reason not recorded: {record['routing_reason']!r}")
|
||||
if record["routing_mode"] != "local":
|
||||
return _fail(f"routing_mode must be local, got {record['routing_mode']!r}")
|
||||
if record["task_family"] != "code":
|
||||
return _fail(f"task_family must be code, got {record['task_family']!r}")
|
||||
return 0
|
||||
|
||||
|
||||
def check_provider_removal_keeps_profiles() -> int:
|
||||
"""Disconnecting a provider (dropping the adapter object) never erases
|
||||
the measured-profile state on disk."""
|
||||
state = Path(_canary_home) / "state"
|
||||
state.mkdir(parents=True, exist_ok=True)
|
||||
profiles_path = state / "model_routing_profiles.json"
|
||||
save_profiles(
|
||||
profiles_path,
|
||||
{
|
||||
("ollama", "qwen3:14b"): SimpleNamespace(
|
||||
per_family_success={"code": [12, 13]},
|
||||
avg_latency_ms=100.0,
|
||||
estimated_cost_usd=0.0,
|
||||
)
|
||||
},
|
||||
)
|
||||
loaded = load_profiles(profiles_path)
|
||||
if list(loaded[("ollama", "qwen3:14b")].per_family_success["code"]) != [12, 13]:
|
||||
return _fail("profiles did not round-trip through disk")
|
||||
|
||||
# "Remove the provider": the adapter instance goes away. A declared
|
||||
# cloud adapter fails loudly (never silently fake).
|
||||
from rune.model_routing import CloudStubAdapter
|
||||
|
||||
try:
|
||||
CloudStubAdapter().chat(model="x", messages=[], tools=None, options={}, think=False)
|
||||
return _fail("declared cloud adapter must raise ProviderUnavailable")
|
||||
except ProviderUnavailable:
|
||||
pass
|
||||
|
||||
# adapter object deleted → profiles file intact and identical
|
||||
if not profiles_path.exists():
|
||||
return _fail("provider removal erased the profiles file")
|
||||
reloaded = load_profiles(profiles_path)
|
||||
if list(reloaded[("ollama", "qwen3:14b")].per_family_success["code"]) != [12, 13]:
|
||||
return _fail("provider removal altered measured profiles")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
checks = (
|
||||
("rank order: cheapest verified-capable first with evidence reasons", check_rank_order),
|
||||
("reason recording: schema-v2 trace carries routing fields", check_reason_recording),
|
||||
("provider removal keeps measured profiles on disk", check_provider_removal_keeps_profiles),
|
||||
)
|
||||
for label, check in checks:
|
||||
result = check()
|
||||
if result != 0:
|
||||
return result
|
||||
print(f" ✓ {label}")
|
||||
print("P1-06 MODEL ROUTING CANARY PASSED")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+16
-314
@@ -1,12 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capability broker — maps user intent to relevant plugins.
|
||||
|
||||
P1-01 (#24): this module is the single capability-selection authority
|
||||
(authority map ``capability_selection`` → v1_owner
|
||||
``ContextBroker.capability_selector``). ``select_for_job`` returns the
|
||||
ordered tool-schema set a job is exposed to; the operator feeds that
|
||||
selection into ``LocalToolRuntime(tools=...)``. Executor authority stays
|
||||
in ``tool_executor`` (#21).
|
||||
Solves the problem: the model must know a plugin exists, know its filename,
|
||||
and know its CLI syntax to use it. The broker removes that requirement.
|
||||
|
||||
Usage:
|
||||
capability_broker.py build # Generate capability cards from registry
|
||||
@@ -31,168 +27,6 @@ REGISTRY = os.path.join(SCRIPTS_DIR, ".microkernel.json")
|
||||
CARDS_FILE = os.path.join(memory_env.memory_dir(), "index", "capability_cards.json")
|
||||
PYTHON = memory_env.python_bin()
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# P1-01 (#24): selection authority — verified floor + escape hatch
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
#: Verified floor: a coding-style job must ALWAYS see these five T1 file
|
||||
#: tools (plan names list_dir/search_files; actual schema names are
|
||||
#: list_directory/grep_files). Selection is never empty and never misses
|
||||
#: the floor.
|
||||
FLOOR_TOOLS = ("read_file", "write_file", "edit_file", "list_directory", "grep_files")
|
||||
|
||||
#: The always-exposed discovery escape hatch (P1-01): the model can name
|
||||
#: any unexposed capability through it and call the discovered tool in
|
||||
#: the SAME run.
|
||||
ESCAPE_HATCH_TOOL = "find_capability"
|
||||
|
||||
#: Upper bound on the exposed set (issue #24: "~5–10").
|
||||
MAX_SELECTED = 10
|
||||
|
||||
|
||||
def _native_tool_schemas():
|
||||
"""Return {tool_name: schema dict} from tool_executor (lazy, safe)."""
|
||||
try:
|
||||
try:
|
||||
from rune.tool_executor import TOOLS
|
||||
except ImportError: # direct script execution
|
||||
from tool_executor import TOOLS
|
||||
except Exception:
|
||||
return {}
|
||||
return {tool["name"]: tool for tool in TOOLS if isinstance(tool.get("name"), str)}
|
||||
|
||||
|
||||
def _infer_task_family(description: str) -> str:
|
||||
"""Task family via the shared #29 inference (coherent with routing)."""
|
||||
try:
|
||||
try:
|
||||
from rune.rune_operator import infer_task_family
|
||||
except ImportError: # direct script execution
|
||||
from rune_operator import infer_task_family
|
||||
return infer_task_family(description)
|
||||
except Exception:
|
||||
return "general"
|
||||
|
||||
|
||||
def _scored_tools(description: str, schemas: dict):
|
||||
"""Score native-tool cards against the task description.
|
||||
|
||||
Deterministic keyword scoring (existing ``score_card``) over the
|
||||
TOOLS-derived cards. Floor tools and the escape hatch are excluded —
|
||||
they are added by policy, not by score. Returns [(score, name)] with
|
||||
score > 0 only, sorted by (-score, name).
|
||||
"""
|
||||
floor = set(FLOOR_TOOLS)
|
||||
scored = []
|
||||
for card in _native_tool_cards():
|
||||
name = card["name"]
|
||||
if name not in schemas or name in floor or name == ESCAPE_HATCH_TOOL:
|
||||
continue
|
||||
score = score_card(card, description)
|
||||
if score > 0:
|
||||
scored.append((score, name))
|
||||
scored.sort(key=lambda pair: (-pair[0], pair[1]))
|
||||
return scored
|
||||
|
||||
|
||||
def select_for_job(task_description, need_tools=True):
|
||||
"""Authoritative capability selection for one job (P1-01 #24).
|
||||
|
||||
Returns ``{"tools": [schema dicts], "names": [ordered names],
|
||||
"reason": str, "task_family": str}``. The ordered selection is the
|
||||
exact schema list the operator feeds into ``LocalToolRuntime(tools=)``
|
||||
— selection flows to the native ``tools`` API param, never the prompt
|
||||
block alone. ``find_capability`` is always exposed (escape hatch).
|
||||
|
||||
Floor guarantee: a tool-using job is never exposed fewer than the
|
||||
five T1 file tools plus the escape hatch, and never an empty set.
|
||||
Zero-score tasks fall back to the floor with reason
|
||||
``"floor (no matches)"``.
|
||||
|
||||
Degrade-not-crash (P0-06): any broker failure falls back to the full
|
||||
TOOLS set (the pre-#24 behavior) and records a tier-1
|
||||
``capability_selection`` failure.
|
||||
"""
|
||||
try:
|
||||
return _select_for_job(str(task_description or ""), bool(need_tools))
|
||||
except Exception as exc: # noqa: BLE001 - degrade, never crash the turn
|
||||
try:
|
||||
from rune.subsystem_health import record_failure
|
||||
|
||||
record_failure("capability_selection", exc, tier=1)
|
||||
except Exception:
|
||||
pass
|
||||
schemas = _native_tool_schemas()
|
||||
names = list(schemas)
|
||||
return {
|
||||
"tools": [schemas[name] for name in names],
|
||||
"names": names,
|
||||
"reason": f"fallback full toolset ({type(exc).__name__})",
|
||||
"task_family": "",
|
||||
}
|
||||
|
||||
|
||||
def _select_for_job(description: str, need_tools: bool) -> dict:
|
||||
schemas = _native_tool_schemas()
|
||||
family = _infer_task_family(description)
|
||||
|
||||
if not need_tools:
|
||||
hatch = schemas.get(ESCAPE_HATCH_TOOL)
|
||||
tools = [hatch] if hatch is not None else []
|
||||
return {
|
||||
"tools": tools,
|
||||
"names": [tool["name"] for tool in tools],
|
||||
"reason": "no tool use expected (escape hatch only)",
|
||||
"task_family": family,
|
||||
}
|
||||
|
||||
floor = [name for name in FLOOR_TOOLS if name in schemas]
|
||||
matches = [name for _, name in _scored_tools(description, schemas)]
|
||||
# Reserve one slot for the escape hatch; cap the whole set.
|
||||
budget = max(0, MAX_SELECTED - len(floor) - 1)
|
||||
matches = matches[:budget]
|
||||
|
||||
ordered = floor + matches
|
||||
hatch = schemas.get(ESCAPE_HATCH_TOOL)
|
||||
if hatch is not None:
|
||||
ordered.append(ESCAPE_HATCH_TOOL)
|
||||
|
||||
if matches:
|
||||
reason = f"floor + {len(matches)} match(es): {', '.join(matches)}"
|
||||
else:
|
||||
reason = "floor (no matches)"
|
||||
|
||||
tools = [schemas[name] for name in ordered if name in schemas]
|
||||
return {
|
||||
"tools": tools,
|
||||
"names": [tool["name"] for tool in tools],
|
||||
"reason": reason,
|
||||
"task_family": family,
|
||||
}
|
||||
|
||||
|
||||
class CapabilityBroker:
|
||||
"""Composition-root handle for the selection authority (P1-01 #24).
|
||||
|
||||
Thin stateful shell over the module-level functions so the
|
||||
composition root can hold exactly one instance per process
|
||||
(mirrors ModelRouter for model_routing). ``cards_dir`` is accepted
|
||||
for composition symmetry; card I/O stays on the module paths.
|
||||
"""
|
||||
|
||||
policy_name = "capability_selection"
|
||||
|
||||
def __init__(self, cards_dir=None):
|
||||
self.cards_dir = cards_dir
|
||||
|
||||
def select_for_job(self, task_description, need_tools: bool = True) -> dict:
|
||||
"""Authoritative selection — see module-level ``select_for_job``."""
|
||||
return select_for_job(task_description, need_tools=need_tools)
|
||||
|
||||
def find_capability(self, query, top_k: int = 5) -> list:
|
||||
"""Escape-hatch query — see module-level ``find_capability``."""
|
||||
return find_capability(query, top_k=top_k)
|
||||
|
||||
# Semantic examples for well-known capabilities
|
||||
SEMANTIC_EXAMPLES = {
|
||||
"memory_recall": [
|
||||
@@ -406,114 +240,6 @@ def load_registry():
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# P1-01 (#24): native-tool card synthesis + load validation
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def _native_tool_cards():
|
||||
"""Synthesize a card per native tool schema from tool_executor.TOOLS.
|
||||
|
||||
tool_executor keeps the single metadata source (risk/preconditions/
|
||||
side_effects); cards copy it so selection has full coverage even with
|
||||
no plugins installed. Lazy import: the broker must be importable where
|
||||
tool_executor is not (memory scripts, hooks).
|
||||
"""
|
||||
try:
|
||||
try:
|
||||
from rune.tool_executor import TOOLS
|
||||
except ImportError: # direct script execution
|
||||
from tool_executor import TOOLS
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
cards = []
|
||||
for tool in TOOLS:
|
||||
name = tool.get("name", "")
|
||||
if not name:
|
||||
continue
|
||||
description = tool.get("description", "")
|
||||
cards.append(
|
||||
{
|
||||
"name": name,
|
||||
"description": description,
|
||||
"capabilities": [name],
|
||||
"file": "tool_executor.py",
|
||||
"full_path": "",
|
||||
"priority": 100,
|
||||
"semantic_examples": [],
|
||||
"risk": tool.get("risk", "read-only"),
|
||||
"preconditions": list(tool.get("preconditions", [])),
|
||||
"side_effects": list(tool.get("side_effects", [])),
|
||||
"auto_invoked": False,
|
||||
"hooks": [],
|
||||
"native_tool": True,
|
||||
"searchable_text": f"{name} {description} native tool".lower(),
|
||||
}
|
||||
)
|
||||
return cards
|
||||
|
||||
|
||||
def _cards_are_stale():
|
||||
"""True when the cards file predates the registry (rebuild trigger).
|
||||
|
||||
A missing registry can never make cards stale — there is nothing
|
||||
newer to rebuild from (native cards are synthesized from TOOLS).
|
||||
"""
|
||||
if not os.path.exists(REGISTRY):
|
||||
return False
|
||||
try:
|
||||
cards_mtime = os.path.getmtime(CARDS_FILE)
|
||||
registry_mtime = os.path.getmtime(REGISTRY)
|
||||
return cards_mtime < registry_mtime
|
||||
except OSError:
|
||||
return True # unreadable cards file → rebuild is the safe move
|
||||
|
||||
|
||||
def load_cards():
|
||||
"""Load capability cards, rebuilding when missing/empty/stale/corrupt.
|
||||
|
||||
P1-01 (#24): the packaged file ships as an empty array — an empty
|
||||
load can no longer pass as a valid card index. Validation rule:
|
||||
rebuild when the file is missing, empty, corrupt JSON, or its mtime
|
||||
is older than the registry it was generated from. Native-tool cards
|
||||
are always synthesized so selection has full tool coverage.
|
||||
"""
|
||||
needs_rebuild = True
|
||||
if os.path.exists(CARDS_FILE):
|
||||
try:
|
||||
with open(CARDS_FILE) as f:
|
||||
existing = json.load(f)
|
||||
needs_rebuild = not isinstance(existing, list) or not existing
|
||||
if not needs_rebuild:
|
||||
needs_rebuild = _cards_are_stale()
|
||||
except (ValueError, OSError):
|
||||
needs_rebuild = True # corrupt file
|
||||
|
||||
if needs_rebuild:
|
||||
cards = build_cards()
|
||||
cards.extend(_native_tool_cards())
|
||||
tmp_file = f"{CARDS_FILE}.tmp"
|
||||
try:
|
||||
os.makedirs(os.path.dirname(CARDS_FILE), exist_ok=True)
|
||||
# Quality discharge (P1-01 #24): atomic rebuild write. A crash
|
||||
# mid-dump must not leave a truncated cards index behind — the
|
||||
# corrupt-JSON rebuild trigger would then mask real generation
|
||||
# bugs. Write a sibling temp file, then atomically rename.
|
||||
with open(tmp_file, "w") as f:
|
||||
json.dump(cards, f, indent=2)
|
||||
os.replace(tmp_file, CARDS_FILE)
|
||||
except OSError:
|
||||
try:
|
||||
os.remove(tmp_file)
|
||||
except OSError:
|
||||
pass # read-only memory dir → selection still works from memory
|
||||
return cards
|
||||
|
||||
with open(CARDS_FILE) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def generate_card(name, manifest):
|
||||
"""Generate a capability card from a plugin manifest."""
|
||||
desc = manifest.get("description", "")
|
||||
@@ -565,8 +291,7 @@ def generate_card(name, manifest):
|
||||
|
||||
|
||||
def build_cards():
|
||||
"""Generate plugin cards from the microkernel registry (native cards
|
||||
are appended by load_cards so plugin-only rebuilds stay pure)."""
|
||||
"""Generate capability cards from the microkernel registry."""
|
||||
reg = load_registry()
|
||||
plugins = reg.get("plugins", {})
|
||||
cards = []
|
||||
@@ -579,9 +304,22 @@ def build_cards():
|
||||
# Sort by priority
|
||||
cards.sort(key=lambda c: c.get("priority", 100))
|
||||
|
||||
# Save
|
||||
os.makedirs(os.path.dirname(CARDS_FILE), exist_ok=True)
|
||||
with open(CARDS_FILE, "w") as f:
|
||||
json.dump(cards, f, indent=2)
|
||||
|
||||
return cards
|
||||
|
||||
|
||||
def load_cards():
|
||||
"""Load capability cards from file."""
|
||||
if not os.path.exists(CARDS_FILE):
|
||||
return build_cards()
|
||||
with open(CARDS_FILE) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def score_card(card, query):
|
||||
"""Score a capability card against a query. Deterministic keyword matching."""
|
||||
query_low = query.lower()
|
||||
@@ -656,42 +394,6 @@ def match_capabilities(query, top_k=5):
|
||||
return [card for _, card in scored[:top_k]]
|
||||
|
||||
|
||||
def find_capability(query, top_k=5):
|
||||
"""Escape-hatch query (P1-01 #24): matching capability summaries.
|
||||
|
||||
Returns up to ``top_k`` matches as dicts carrying name, description,
|
||||
risk, and — for native tools — the full ``parameters`` schema so the
|
||||
model can call the discovered tool in the same session. Plugin
|
||||
capabilities without a native schema come back as name+description
|
||||
only (the runtime cannot expose what has no executor).
|
||||
"""
|
||||
query = str(query or "")
|
||||
schemas = _native_tool_schemas()
|
||||
cards = load_cards()
|
||||
scored = []
|
||||
for card in cards:
|
||||
name = card.get("name", "")
|
||||
if not name or name in FLOOR_TOOLS or name == ESCAPE_HATCH_TOOL:
|
||||
continue
|
||||
score = score_card(card, query)
|
||||
if score > 0:
|
||||
scored.append((score, name, card))
|
||||
scored.sort(key=lambda item: (-item[0], item[1]))
|
||||
|
||||
out = []
|
||||
for _, name, card in scored[:top_k]:
|
||||
schema = schemas.get(name)
|
||||
entry = {
|
||||
"name": name,
|
||||
"description": card.get("description", ""),
|
||||
"risk": card.get("risk", "read-only"),
|
||||
}
|
||||
if schema is not None:
|
||||
entry["parameters"] = schema.get("parameters", {})
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
def handle_hook(event, data=None):
|
||||
"""On-message hook: find relevant capabilities for the user's message."""
|
||||
if event == "on_message" and data:
|
||||
|
||||
@@ -79,18 +79,10 @@ class ContributionBlock:
|
||||
self.evictable = evictable # False = never dropped (system, user)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Serialize for pending_state embedding (approval resume).
|
||||
|
||||
P1-02 (#25): ``content`` is included so the embedded capsule is
|
||||
verifiable end-to-end — the acceptance criterion is that the logged
|
||||
capsule contents EXACTLY equal the model payload, which token counts
|
||||
alone cannot prove. Blocks are small (admission-bounded), so the
|
||||
extra size is bounded by the same budget.
|
||||
"""
|
||||
"""Serialize for pending_state embedding (approval resume)."""
|
||||
return {
|
||||
"name": self.name,
|
||||
"priority": self.priority,
|
||||
"content": self.content,
|
||||
"tokens": self.tokens,
|
||||
"source": self.source,
|
||||
"evictable": self.evictable,
|
||||
|
||||
@@ -1,527 +0,0 @@
|
||||
"""Failure auto-filer (#122): subsystem failures become forge issues.
|
||||
|
||||
Detect → record (subsystem_health, untouched) → track (the forge issue) →
|
||||
fix. One forge issue per failure signature ``(subsystem, error_class)``;
|
||||
repeats comment with a rising count; T2 (ambient/cron) failures stay
|
||||
journal-only — cron noise must never become board noise; any forge
|
||||
exception spools to the durable event journal and is reconciled on
|
||||
recovery. The filer NEVER raises into the failure path.
|
||||
|
||||
Threading: the failure path only ``submit()``s (enqueue, no network) and
|
||||
a single daemon worker drains the queue, so failure-path latency is
|
||||
untouched. The worker coalesces bursts: everything already queued for a
|
||||
signature is batched into ONE client write (first occurrence files,
|
||||
repeats comment once per ``batch_size`` occurrences or 10 minutes,
|
||||
whichever first). ``file()`` is the synchronous single-occurrence
|
||||
pipeline step (used by tests and reconcile); the tier gate lives here
|
||||
and nowhere else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from rune.forge_client import ForgeClient
|
||||
|
||||
AUTOFILE_OFF_ENV = "RUNE_AUTOFILE_OFF"
|
||||
SPOOL_KIND = "failure_auto_file_spool"
|
||||
BATCH_INTERVAL_SECONDS = 600 # one comment per 10 occurrences OR 10 minutes
|
||||
_EXCERPT_LEN = 200
|
||||
_ERROR_CLASS_MAX = 64 # a 10KB single-token error must not make a 10KB marker
|
||||
_QUEUE_MAXSIZE = 1000 # bounded: an outage must not balloon failure-path memory
|
||||
|
||||
_LOG = logging.getLogger("rune.failure_auto_filer")
|
||||
|
||||
|
||||
def error_class_of(error: BaseException | str) -> str:
|
||||
"""Exception class name, or the first whitespace token of a string
|
||||
(trailing ``:`` stripped, so ``"RuntimeError: boom"`` → ``RuntimeError``).
|
||||
Capped at 64 chars (#122 gate): an unbounded single-token error would
|
||||
make an unbounded marker."""
|
||||
if isinstance(error, BaseException):
|
||||
return type(error).__name__[:_ERROR_CLASS_MAX]
|
||||
tokens = str(error).split()
|
||||
return (tokens[0].rstrip(":") if tokens else "Unknown")[:_ERROR_CLASS_MAX]
|
||||
|
||||
|
||||
def signature_of(subsystem: str, error: BaseException | str) -> tuple[str, str]:
|
||||
"""The dedup signature: (subsystem, error_class). Never the full
|
||||
message — variable payloads would flood the board."""
|
||||
return (subsystem, error_class_of(error))
|
||||
|
||||
|
||||
def tier_gated(tier: int) -> bool:
|
||||
"""THE tier gate — the ONE place (module-level so the live path AND
|
||||
the spool replay share it): T0/T1 file/comment, T2 journal-only
|
||||
(cron noise must never become board noise)."""
|
||||
return int(tier) >= 2
|
||||
|
||||
|
||||
def marker_for(subsystem: str, error_class: str) -> str:
|
||||
return f"[auto-failure:{subsystem}:{error_class}]"
|
||||
|
||||
|
||||
def excerpt(error: BaseException | str, limit: int = _EXCERPT_LEN) -> str:
|
||||
"""The excerpt that lands in PERSISTENT forge issues and spool lines:
|
||||
redacted first (broker exceptions can embed credentialed URLs/DSNs),
|
||||
then truncated. Error class and shape stay visible."""
|
||||
return redact(str(error))[:limit]
|
||||
|
||||
|
||||
# Credentials must never persist in forge issues. Patterns (order matters
|
||||
# little; all are applied): query-string credential params, URL userinfo
|
||||
# (scheme://user:pass@host — covers postgres/redis/mysql DSNs and basic
|
||||
# auth), Authorization headers / bearer tokens, and assignment forms
|
||||
# (password=..., secret=...). Only the credential SPAN is replaced — the
|
||||
# surrounding error text (class, host, port, message shape) survives.
|
||||
_QUERY_CREDS = re.compile(
|
||||
r"([?&](?:token|password|passwd|secret|api_key|apikey|access_token|key|sig|signature|credential)=)[^&\s'\"]+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_URL_USERINFO = re.compile(r"\b([a-zA-Z][a-zA-Z0-9+.-]*://)([^\s:/@'\"]+)?:([^\s/@'\"]+)@")
|
||||
_BEARER = re.compile(r"(?i)\b((?:authorization\s*:?\s*(?:bearer\s+)?|bearer\s+))[^\s,'\";]+")
|
||||
_ASSIGNMENT = re.compile(
|
||||
r"(?i)\b((?:password|passwd|secret|api_key|apikey|access_token|auth)\s*[=:]\s*)['\"]?([^\s'\"]{4,})"
|
||||
)
|
||||
|
||||
|
||||
def redact(text: str) -> str:
|
||||
"""Strip credentials from an error string (no-op on clean text)."""
|
||||
out = _QUERY_CREDS.sub(r"\1<redacted>", text)
|
||||
out = _URL_USERINFO.sub(r"\1<redacted>:<redacted>@", out)
|
||||
out = _BEARER.sub(r"\1<redacted>", out)
|
||||
out = _ASSIGNMENT.sub(lambda m: f"{m.group(1)}<redacted>", out)
|
||||
return out
|
||||
|
||||
|
||||
def _error_text(error: BaseException | str) -> str:
|
||||
return str(error) if isinstance(error, str) else f"{type(error).__name__}: {error}"
|
||||
|
||||
|
||||
class SpoolJournal(Protocol):
|
||||
"""What the filer needs from a durable journal (satisfied by
|
||||
JsonlSpool over the existing event journal file)."""
|
||||
|
||||
def append(self, record: dict) -> None: ...
|
||||
def unconsumed(self) -> list[dict]: ...
|
||||
def mark_consumed(self, spool_id: str) -> None: ...
|
||||
|
||||
|
||||
class JsonlSpool:
|
||||
"""Spool riding the EXISTING durable event journal (no new storage).
|
||||
|
||||
Spooled records are appended as JSON lines with ``kind:
|
||||
failure_auto_file_spool`` next to the canonical envelopes (which have
|
||||
``event_id`` keys). The canonical dialect is guarded on
|
||||
DurableEventStore.append only, so co-located foreign lines are safe.
|
||||
|
||||
Consumption NEVER rewrites the canonical journal (#122 gate): the
|
||||
old read-modify-write raced concurrent DurableEventStore.appends
|
||||
(an O_APPEND landing between the read and the write was silently
|
||||
destroyed) and a crash mid-rewrite poisoned the canonical store.
|
||||
Instead the marker is an append-only sidecar ``<journal>.consumed``
|
||||
(one spool id per line, atomic appends only, O(1) per mark — no
|
||||
O(N) rewrite). ``unconsumed()`` = spool lines minus sidecar ids.
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = Path(path)
|
||||
|
||||
def append(self, record: dict) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.path, "a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(record, sort_keys=True) + "\n")
|
||||
|
||||
def unconsumed(self) -> list[dict]:
|
||||
if not self.path.exists():
|
||||
return []
|
||||
sidecar: set[str] = set()
|
||||
sidecar_path = Path(f"{self.path}.consumed")
|
||||
if sidecar_path.exists():
|
||||
sidecar = {line.strip() for line in sidecar_path.read_text(encoding="utf-8").splitlines() if line.strip()}
|
||||
rows: list[dict] = []
|
||||
for line in self.path.read_text(encoding="utf-8").splitlines():
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if (
|
||||
isinstance(row, dict)
|
||||
and row.get("kind") == SPOOL_KIND
|
||||
and not row.get("consumed")
|
||||
and row.get("spool_id") not in sidecar
|
||||
):
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
def mark_consumed(self, spool_id: str) -> None:
|
||||
"""Record consumption in the append-only sidecar — the canonical
|
||||
journal is NEVER rewritten (append-only, crash-safe, race-free
|
||||
with concurrent DurableEventStore.appends)."""
|
||||
try:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(f"{self.path}.consumed", "a", encoding="utf-8") as fh:
|
||||
fh.write(spool_id + "\n")
|
||||
except OSError:
|
||||
pass # last resort: the line stays unconsumed; dedup via
|
||||
# search_issues still prevents duplicate issues on replay.
|
||||
|
||||
|
||||
class _SigState:
|
||||
"""In-memory, process-lifetime coalescing state per signature."""
|
||||
|
||||
__slots__ = ("count", "pending", "window_start")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.count = 0 # total occurrences seen this process
|
||||
self.pending = 0 # occurrences since the last client write
|
||||
self.window_start = 0.0 # monotonic time of the last client write
|
||||
|
||||
|
||||
class FailureAutoFiler:
|
||||
"""One issue per failure signature; tier gate + spool + worker drain.
|
||||
|
||||
The tier gate (tier in (0, 1) → file/comment, tier >= 2 → journal
|
||||
only) is enforced HERE and nowhere else.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: ForgeClient,
|
||||
journal: SpoolJournal,
|
||||
batch_size: int = 10,
|
||||
wait: bool = False,
|
||||
) -> None:
|
||||
self._client = client
|
||||
self._journal = journal
|
||||
self._queue: queue.Queue[tuple[str, str, int, float]] = queue.Queue(maxsize=_QUEUE_MAXSIZE)
|
||||
self._lock = threading.Lock()
|
||||
self._state: dict[tuple[str, str], _SigState] = {}
|
||||
self._batch_size = max(1, batch_size)
|
||||
self._worker: threading.Thread | None = None
|
||||
self._inflight: set[tuple[str, str]] = set() # single-flight per signature
|
||||
if not wait:
|
||||
self._worker = threading.Thread(target=self._run_worker, name="rune-auto-filer", daemon=True)
|
||||
self._worker.start()
|
||||
|
||||
# ---------------------------------------------------- failure-path API
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
subsystem: str,
|
||||
error: BaseException | str,
|
||||
tier: int,
|
||||
_now: float | None = None,
|
||||
) -> None:
|
||||
"""Callable-filer contract: record_failure invokes the wired
|
||||
filer directly; this is exactly submit()."""
|
||||
self.submit(subsystem, error, tier, _now)
|
||||
|
||||
def submit(
|
||||
self,
|
||||
subsystem: str,
|
||||
error: BaseException | str,
|
||||
tier: int,
|
||||
_now: float | None = None,
|
||||
) -> None:
|
||||
"""The failure-path entry: enqueue and return immediately.
|
||||
|
||||
Never blocks on the network, never raises (swallowed + logged).
|
||||
With ``wait=True`` (synchronous/test mode) nothing is written
|
||||
until ``drain()`` — the whole backlog coalesces into one write
|
||||
per signature.
|
||||
"""
|
||||
try:
|
||||
if tier_gated(tier):
|
||||
return # THE tier gate — T2 journal-only, zero queue traffic
|
||||
now = time.monotonic() if _now is None else _now
|
||||
try:
|
||||
self._queue.put_nowait((subsystem, _error_text(error), int(tier), now))
|
||||
except queue.Full:
|
||||
# Bounded queue (#122 gate): an overflow occurrence is
|
||||
# dropped from the queue but SPOOLED — nothing is lost —
|
||||
# and the drop is logged. Never blocks the failure path.
|
||||
_LOG.warning("auto-filer queue full; spooling dropped occurrence for subsystem=%s", subsystem)
|
||||
self._spool(subsystem, error, tier, Exception("auto-filer queue full"), count=1)
|
||||
except Exception: # noqa: BLE001 - never raise into the failure path
|
||||
_LOG.exception("auto-filer submit failed for subsystem=%s", subsystem)
|
||||
|
||||
def drain(self, timeout: float | None = None) -> None:
|
||||
"""Block until the queue is empty (tests + operator shutdown)."""
|
||||
if self._worker is None:
|
||||
self._drain_once()
|
||||
return
|
||||
deadline = None if timeout is None else time.monotonic() + timeout
|
||||
while self._queue.unfinished_tasks > 0:
|
||||
if deadline is not None and time.monotonic() > deadline:
|
||||
return
|
||||
time.sleep(0.005)
|
||||
|
||||
# -------------------------------------------------- synchronous pipeline
|
||||
|
||||
def file(
|
||||
self,
|
||||
subsystem: str,
|
||||
error: BaseException | str,
|
||||
tier: int,
|
||||
_now: float | None = None,
|
||||
) -> bool:
|
||||
"""Synchronous single-occurrence pipeline step.
|
||||
|
||||
Tier gate → coalescing state → (first occurrence, batch full, or
|
||||
window expired) → search → create-or-comment. ANY client
|
||||
exception is spooled and swallowed; this method never raises.
|
||||
Returns True when a forge write was issued.
|
||||
"""
|
||||
try:
|
||||
if tier_gated(tier):
|
||||
return False # THE tier gate — one place, T2 journal-only
|
||||
now = time.monotonic() if _now is None else _now
|
||||
sig = signature_of(subsystem, error)
|
||||
text = _error_text(error)
|
||||
with self._lock:
|
||||
state = self._state.setdefault(sig, _SigState())
|
||||
state.count += 1
|
||||
state.pending += 1
|
||||
due = (
|
||||
state.count == 1
|
||||
or state.pending >= self._batch_size
|
||||
or (state.pending > 0 and now - state.window_start >= BATCH_INTERVAL_SECONDS)
|
||||
)
|
||||
count_at_flush = state.count
|
||||
if due:
|
||||
state.pending = 0
|
||||
state.window_start = now
|
||||
if not due:
|
||||
return False # coalesced: no client call for this occurrence
|
||||
return self._write(sig, text, int(tier), count_at_flush)
|
||||
except Exception as exc: # noqa: BLE001 - degrade, never propagate
|
||||
self._spool(subsystem, error, tier, exc)
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------- reconcile
|
||||
|
||||
def reconcile(self) -> dict[str, int]:
|
||||
"""Replay unconsumed spool lines through the real pipeline.
|
||||
|
||||
Lines are GROUPED by signature first (#122 gate): one client
|
||||
write per signature per reconcile pass, with the aggregated
|
||||
offline count (a long outage must not comment-flood the issue).
|
||||
Dedup re-applies via search_issues (a line whose issue already
|
||||
exists becomes one comment, never a duplicate issue). Confirmed-
|
||||
filed lines are marked consumed; failures leave the line spooled.
|
||||
Never raises. Called at operator startup (the maintenance
|
||||
cadence — see the plan-tail deviation, #122).
|
||||
"""
|
||||
report = {"replayed": 0, "filed": 0, "failed": 0, "skipped": 0}
|
||||
try:
|
||||
pending = self._journal.unconsumed()
|
||||
except Exception: # noqa: BLE001
|
||||
_LOG.exception("reconcile could not read the spool journal")
|
||||
report["failed"] += 1
|
||||
return report
|
||||
groups: dict[tuple[str, str], list[dict]] = {}
|
||||
order: list[tuple[str, str]] = []
|
||||
for record in pending:
|
||||
report["replayed"] += 1
|
||||
signature = record.get("signature") or []
|
||||
if len(signature) != 2:
|
||||
report["failed"] += 1 # malformed line: leave it, never crash
|
||||
continue
|
||||
if tier_gated(int(record.get("tier", 1))):
|
||||
# THE tier gate on replay — the same predicate as the live
|
||||
# path (one place). A T2 line resolves journal-only:
|
||||
# consumed, never written.
|
||||
self._journal.mark_consumed(str(record.get("spool_id", "")))
|
||||
report["skipped"] += 1
|
||||
continue
|
||||
sig = (str(signature[0]), str(signature[1]))
|
||||
if sig not in groups:
|
||||
groups[sig] = []
|
||||
order.append(sig)
|
||||
groups[sig].append(record)
|
||||
for sig in order:
|
||||
group = groups[sig]
|
||||
last = group[-1]
|
||||
try:
|
||||
filed = self._write(
|
||||
sig,
|
||||
str(last.get("error", "")),
|
||||
int(last.get("tier", 1)),
|
||||
sum(max(1, int(r.get("count", 1))) for r in group),
|
||||
offline=len(group),
|
||||
)
|
||||
if filed:
|
||||
for record in group:
|
||||
self._journal.mark_consumed(str(record.get("spool_id", "")))
|
||||
report["filed"] += len(group)
|
||||
else:
|
||||
report["failed"] += len(group) # e.g. NullAdapter: nothing was filed
|
||||
except Exception: # noqa: BLE001 - forge still down: keep spooled
|
||||
report["failed"] += len(group)
|
||||
return report
|
||||
|
||||
# ------------------------------------------------------------ internals
|
||||
|
||||
def _write(self, sig: tuple[str, str], error_text: str, tier: int, count: int, offline: int = 0) -> bool:
|
||||
"""One client write for a signature: comment on the existing issue
|
||||
(dedup via server-side search) or create it. ``offline`` > 0 marks
|
||||
a reconcile replay of ``offline`` spooled lines (aggregated "while
|
||||
offline" count). Raises on client failure — callers spool.
|
||||
|
||||
Single-flight per signature (#122 gate): the reconcile thread and
|
||||
the worker share an in-flight set guarded by the class lock; a
|
||||
racing caller for a signature already being written does NOT
|
||||
write (returns False) — its spool line stays spooled and dedup
|
||||
re-applies on the next replay. Cross-PROCESS dedup remains
|
||||
best-effort TOCTOU between search and create (documented
|
||||
limitation: two rune PROCESSES can still double-file).
|
||||
"""
|
||||
with self._lock:
|
||||
if sig in self._inflight:
|
||||
return False # single-flight: another thread owns this write
|
||||
self._inflight.add(sig)
|
||||
try:
|
||||
return self._write_once(sig, error_text, tier, count, offline)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._inflight.discard(sig)
|
||||
|
||||
def _write_once(self, sig: tuple[str, str], error_text: str, tier: int, count: int, offline: int = 0) -> bool:
|
||||
subsystem, error_class = sig
|
||||
marker = marker_for(subsystem, error_class)
|
||||
existing = self._client.search_issues(marker)
|
||||
match = next(
|
||||
(i for i in existing if marker in str(i.get("title", ""))),
|
||||
None,
|
||||
)
|
||||
ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
if match is not None:
|
||||
if offline > 0:
|
||||
body = f"+{count} occurrences while offline (last: {excerpt(error_text)} @ {ts})"
|
||||
else:
|
||||
body = f"+{max(1, count - 1)} occurrences (last: {excerpt(error_text)} @ {ts})"
|
||||
self._client.add_comment(int(match["number"]), body)
|
||||
return True
|
||||
title = f"{marker} {subsystem}"
|
||||
body = (
|
||||
f"Auto-filed failure signature `{subsystem}:{error_class}`\n\n"
|
||||
f"- tier={tier}\n"
|
||||
f"- occurrences={count}\n"
|
||||
f"- last: {excerpt(error_text)}\n\n"
|
||||
f"(detected by the failure auto-filer, #122)"
|
||||
)
|
||||
result = self._client.create_issue(title, body)
|
||||
if isinstance(result, dict) and result.get("created") is False:
|
||||
return False # e.g. NullAdapter: nothing was filed
|
||||
return True
|
||||
|
||||
def _spool(
|
||||
self,
|
||||
subsystem: str,
|
||||
error: BaseException | str,
|
||||
tier: int,
|
||||
exc: Exception,
|
||||
count: int = 1,
|
||||
) -> None:
|
||||
"""Persist the failure for reconcile(). Never raises."""
|
||||
try:
|
||||
subsystem_name, error_class = signature_of(subsystem, error)
|
||||
record = {
|
||||
"kind": SPOOL_KIND,
|
||||
"spool_id": f"spool-{time.time_ns()}",
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"signature": [subsystem_name, error_class],
|
||||
"tier": int(tier),
|
||||
"count": int(count),
|
||||
"error": excerpt(error),
|
||||
"client_error": excerpt(exc),
|
||||
}
|
||||
self._journal.append(record)
|
||||
except Exception: # noqa: BLE001 - even the spool must not raise
|
||||
_LOG.exception("failed to spool auto-file record for subsystem=%s", subsystem)
|
||||
|
||||
# ------------------------------------------------------------- worker
|
||||
|
||||
def _run_worker(self) -> None:
|
||||
while True:
|
||||
item = self._queue.get()
|
||||
if item is None: # shutdown sentinel
|
||||
self._queue.task_done()
|
||||
return
|
||||
batch = [item]
|
||||
while True: # coalesce: everything already queued rides one write
|
||||
try:
|
||||
batch.append(self._queue.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
try:
|
||||
self._process_batch(batch)
|
||||
except Exception: # noqa: BLE001 - worker must never die
|
||||
_LOG.exception("auto-filer worker batch failed")
|
||||
finally:
|
||||
for _ in batch:
|
||||
self._queue.task_done()
|
||||
|
||||
def _process_batch(self, batch: list[tuple[str, str, int, float]]) -> None:
|
||||
groups: dict[tuple[str, str], list[tuple[str, str, int, float]]] = {}
|
||||
order: list[tuple[str, str]] = []
|
||||
for subsystem, text, tier, now in batch:
|
||||
if tier_gated(tier):
|
||||
continue # THE tier gate (defense in depth; submit gates too)
|
||||
sig = signature_of(subsystem, text)
|
||||
if sig not in groups:
|
||||
groups[sig] = []
|
||||
order.append(sig)
|
||||
groups[sig].append((subsystem, text, tier, now))
|
||||
for sig in order:
|
||||
group = groups[sig]
|
||||
try:
|
||||
self._flush_group(sig, group)
|
||||
except Exception as exc: # noqa: BLE001 - spool, never propagate
|
||||
for subsystem, text, tier, _now in group:
|
||||
self._spool(subsystem, text, tier, exc, count=len(group))
|
||||
|
||||
def _flush_group(self, sig: tuple[str, str], group: list[tuple[str, str, int, float]]) -> None:
|
||||
now = group[-1][3]
|
||||
text = group[-1][1] # last error of the batch
|
||||
tier = group[-1][2]
|
||||
with self._lock:
|
||||
state = self._state.setdefault(sig, _SigState())
|
||||
state.count += len(group)
|
||||
state.pending += len(group)
|
||||
due = (
|
||||
state.count == len(group)
|
||||
or state.pending >= self._batch_size
|
||||
or (state.pending > 0 and now - state.window_start >= BATCH_INTERVAL_SECONDS)
|
||||
)
|
||||
count_at_flush = state.count
|
||||
if due:
|
||||
state.pending = 0
|
||||
state.window_start = now
|
||||
if due:
|
||||
self._write(sig, text, tier, count_at_flush)
|
||||
|
||||
def _drain_once(self) -> None:
|
||||
"""Inline drain for synchronous (wait=True) mode."""
|
||||
batch: list[tuple[str, str, int, float]] = []
|
||||
while True:
|
||||
try:
|
||||
batch.append(self._queue.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
if batch:
|
||||
try:
|
||||
self._process_batch(batch)
|
||||
except Exception: # noqa: BLE001 - must never escape
|
||||
_LOG.exception("auto-filer inline drain failed")
|
||||
finally:
|
||||
for _ in batch:
|
||||
self._queue.task_done()
|
||||
@@ -20,7 +20,6 @@ class ForgeClient(Protocol):
|
||||
def add_comment(self, issue_number: int, body: str) -> dict: ...
|
||||
def get_issue_state(self, issue_number: int) -> dict: ...
|
||||
def list_open_issues(self) -> list[dict]: ...
|
||||
def search_issues(self, query: str) -> list[dict]: ...
|
||||
|
||||
|
||||
class NullAdapter:
|
||||
@@ -38,9 +37,6 @@ class NullAdapter:
|
||||
def list_open_issues(self) -> list[dict]:
|
||||
return []
|
||||
|
||||
def search_issues(self, query: str) -> list[dict]:
|
||||
return []
|
||||
|
||||
|
||||
class GiteaForgejoAdapter:
|
||||
"""Gitea/Forgejo adapter — /api/v1 paths (identical on both platforms)."""
|
||||
@@ -96,37 +92,6 @@ class GiteaForgejoAdapter:
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def search_issues(self, query: str) -> list[dict]:
|
||||
"""Repo-scoped marker search over the paginated list endpoint.
|
||||
|
||||
#122 gate fix: Gitea/Forgejo has NO repo-scoped ``/issues/search``
|
||||
path (verified against live swagger) — that URL 404s on a real
|
||||
forge, so every dedup search would raise and everything would
|
||||
spool forever. The repo-scoped LIST endpoint
|
||||
(``GET /repos/{owner}/{repo}/issues``) supports ``state``, ``type``
|
||||
and ``q`` server-side and is paginated: ``q`` matches titles (and
|
||||
keywords), so the full marker is sent as ``q`` and a client-side
|
||||
contains-check keeps exactly the issues whose title OR body
|
||||
contains the marker (dedup must see body mentions too, but a
|
||||
title hit is what the filer matches on).
|
||||
"""
|
||||
results: list[dict] = []
|
||||
page = 1
|
||||
while True:
|
||||
from urllib.parse import urlencode
|
||||
|
||||
r = httpx.get(
|
||||
f"{self._base_url}/issues?{urlencode({'state': 'open', 'type': 'issues', 'q': query, 'limit': 50, 'page': page})}",
|
||||
headers=self._headers(),
|
||||
timeout=30,
|
||||
)
|
||||
r.raise_for_status()
|
||||
batch = r.json()
|
||||
results.extend(batch)
|
||||
if len(batch) < 50:
|
||||
return [i for i in results if query in str(i.get("title", "")) + str(i.get("body", ""))]
|
||||
page += 1
|
||||
|
||||
|
||||
def create_forge_client(config: dict | None = None) -> ForgeClient:
|
||||
"""Create a forge client from config or environment.
|
||||
|
||||
@@ -17,7 +17,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
@@ -25,7 +25,7 @@ try:
|
||||
except ImportError: # direct script execution
|
||||
from rune.memory_env import rune_home
|
||||
|
||||
INFERENCE_TRACE_SCHEMA_VERSION = 4
|
||||
INFERENCE_TRACE_SCHEMA_VERSION = 1
|
||||
TRACE_ROTATE_BYTES = 10_000_000
|
||||
TRACE_BACKUPS = 3
|
||||
|
||||
@@ -78,22 +78,6 @@ class InferenceTrace:
|
||||
cost_basis: str # "local-unpriced" | "price-map"
|
||||
success: bool
|
||||
error: str
|
||||
# P1-06 (#29) schema v2 — additive optional fields. Defaults keep v1
|
||||
# readers and v1 rows fully compatible (missing keys on old records).
|
||||
routing_reason: str = "" # the start-tier decision reason (router)
|
||||
routing_mode: str = "" # "local" | "cloud" | "" (pre-router rows)
|
||||
task_family: str = "" # measured-registry family key
|
||||
# P1-01 (#24) schema v3 — capability-authority fields, additive.
|
||||
selected_tools: list = field(default_factory=list) # ordered exposed set
|
||||
capability_reason: str = "" # broker selection reason ("" = unselected)
|
||||
find_capability_calls: int = 0 # escape-hatch round-trips this run
|
||||
# P1-02 (#25) schema v4 — capsule-fidelity fields, additive. The logged
|
||||
# capsule contents must EXACTLY equal the bounded dynamic context
|
||||
# supplied to the model; message list (the actual payload) + per-block
|
||||
# content SHA-256 (integrity without full JSONL bloat) prove it.
|
||||
capsule_messages: list = field(default_factory=list) # exact runtime.run payload
|
||||
capsule_content_hashes: dict = field(default_factory=dict) # block name -> sha256
|
||||
capsule_token_count: int = 0 # post-decoration re-measured total
|
||||
|
||||
|
||||
def price_for(model: str) -> float | None:
|
||||
|
||||
@@ -19,7 +19,6 @@ from .tool_executor import (
|
||||
serialize_prepared_tool,
|
||||
)
|
||||
from .tool_protocol import (
|
||||
TOOL_INDEX,
|
||||
OperatorResponse,
|
||||
ToolEvent,
|
||||
ToolRequest,
|
||||
@@ -46,23 +45,6 @@ _CONTEXT_LENGTH_LOCK = threading.Lock()
|
||||
#: Fallback tool prompt is runtime-owned but traced — once per session.
|
||||
_FALLBACK_PROMPT_TRACED: set[str] = set()
|
||||
|
||||
# P1-01 (#24): the capability escape hatch. Executing ``find_capability``
|
||||
# queries the broker and returns matching capability summaries; the
|
||||
# runtime then auto-exposes the discovered native tools for the rest of
|
||||
# the run (within the existing iteration budget — no extra turns granted).
|
||||
FIND_CAPABILITY_TOOL = "find_capability"
|
||||
|
||||
|
||||
def _find_capability_executor(tool_name: str, arguments: dict) -> dict:
|
||||
"""Default escape-hatch executor: route through tool_executor.
|
||||
|
||||
Kept as a module-level wrapper so tests and callers can exercise the
|
||||
hatch without constructing the full runtime.
|
||||
"""
|
||||
from .tool_executor import execute_tool
|
||||
|
||||
return execute_tool(tool_name, arguments)
|
||||
|
||||
|
||||
class ChatMessage(TypedDict, total=False):
|
||||
"""Ollama chat message fields used by the runtime."""
|
||||
@@ -294,12 +276,6 @@ class LocalToolRuntime:
|
||||
definition["name"] for definition in self.tools if isinstance(definition.get("name"), str)
|
||||
)
|
||||
self.max_iterations = max_iterations
|
||||
# P1-01 (#24): escape-hatch executor. The runtime's own executor
|
||||
# stays authoritative for every tool EXCEPT find_capability, whose
|
||||
# result drives dynamic in-run exposure (a plain executor callable
|
||||
# cannot mutate the tool set).
|
||||
self.find_capability_executor = _find_capability_executor
|
||||
self.find_capability_calls = 0
|
||||
|
||||
def run(
|
||||
self,
|
||||
@@ -344,13 +320,6 @@ class LocalToolRuntime:
|
||||
raise ValueError("pending think must be a boolean or string")
|
||||
think = raw_think
|
||||
pending_request = self._request_from_state(copy.deepcopy(pending_state["pending_request"]))
|
||||
# P1-01 #24 quality discharge: honor the embedded exposed set.
|
||||
# Additive + backward compatible (pre-#24 states lack the key);
|
||||
# only names with real schemas are exposed, never invented.
|
||||
raw_exposed = pending_state.get("exposed_tools", [])
|
||||
if not isinstance(raw_exposed, list) or not all(isinstance(name, str) for name in raw_exposed):
|
||||
raise ValueError("pending exposed tools must be a list of strings")
|
||||
self._expose_tools(raw_exposed)
|
||||
if pending_request.name not in self.tool_names:
|
||||
raise ValueError(f"tool not available in this runtime: {pending_request.name}")
|
||||
raw_events = copy.deepcopy(pending_state.get("tool_events", []))
|
||||
@@ -453,17 +422,8 @@ class LocalToolRuntime:
|
||||
if requests:
|
||||
for index, request in enumerate(requests):
|
||||
if request.name not in self.tool_names:
|
||||
# P1-01 (#24): a tool outside this runtime's allowlist
|
||||
# is rejected with one-step recovery guidance naming
|
||||
# the escape hatch. Dynamic exposure happens ONLY
|
||||
# through find_capability results (see the hatch
|
||||
# dispatch below) — never by naming a tool directly,
|
||||
# which would bypass broker selection.
|
||||
return self._failure(
|
||||
f"tool not available in this runtime: {request.name}. "
|
||||
f"If you need a capability that is not exposed, call "
|
||||
f"{FIND_CAPABILITY_TOOL} with a description of what "
|
||||
f"you need to do.",
|
||||
f"tool not available in this runtime: {request.name}",
|
||||
events,
|
||||
thinking,
|
||||
iterations_used=iterations_used,
|
||||
@@ -511,13 +471,6 @@ class LocalToolRuntime:
|
||||
"tool_events": [self._event_to_state(event) for event in events],
|
||||
"thinking": copy.deepcopy(thinking),
|
||||
"remaining_requests": [self._request_to_state(item) for item in requests[index + 1 :]],
|
||||
# P1-01 #24 quality discharge: the run's full
|
||||
# exposed set (selected + any find_capability
|
||||
# discoveries) rides the state so a FRESH
|
||||
# runtime instance can resume it — without
|
||||
# this, resume() on a newly constructed runtime
|
||||
# rejects a dynamically exposed tool.
|
||||
"exposed_tools": sorted(self.tool_names),
|
||||
}
|
||||
if prepared is not None:
|
||||
state["prepared_tool"] = serialize_prepared_tool(prepared)
|
||||
@@ -532,28 +485,7 @@ class LocalToolRuntime:
|
||||
iteration_budget=self.max_iterations,
|
||||
)
|
||||
try:
|
||||
if request.name == FIND_CAPABILITY_TOOL:
|
||||
# P1-01 (#24): the hatch executes through its
|
||||
# dedicated executor (a plain executor callable
|
||||
# cannot mutate the tool set to expose matches);
|
||||
# a successful query then exposes the returned
|
||||
# native tools in-run. Quality discharge: the
|
||||
# call rides the SAME validated dispatch as
|
||||
# every other tool — find_capability has a real
|
||||
# schema in TOOL_INDEX, so resumed requests
|
||||
# (which skip intake validation) cannot smuggle
|
||||
# arguments past the handler; validation errors
|
||||
# surface as tool errors via the except below.
|
||||
result = dict(
|
||||
dispatch_validated_tool(request, self.find_capability_executor)
|
||||
)
|
||||
if result.get("success"):
|
||||
self.find_capability_calls += 1
|
||||
self._expose_tools(result.get("exposed") or [])
|
||||
if native_tools:
|
||||
native_definitions = to_ollama_tools(self.tools)
|
||||
else:
|
||||
result = dispatch_validated_tool(request, self.executor)
|
||||
result = dispatch_validated_tool(request, self.executor)
|
||||
except Exception as exc:
|
||||
result = {"success": False, "error": f"executor error: {exc}"}
|
||||
|
||||
@@ -597,26 +529,11 @@ class LocalToolRuntime:
|
||||
raise ValueError("transport response is missing a message object")
|
||||
assistant = cast(ChatMessage, assistant_object)
|
||||
assistant = cast(ChatMessage, copy.deepcopy(assistant))
|
||||
requests = []
|
||||
try:
|
||||
requests = (
|
||||
normalize_ollama_tool_calls(cast(dict, assistant))
|
||||
if native_tools
|
||||
else normalize_text_tool_calls(assistant.get("content", ""))
|
||||
)
|
||||
except ValueError as exc:
|
||||
if str(exc).startswith("unknown tool: "):
|
||||
# P1-01 (#24): one-step recovery — name the escape
|
||||
# hatch so the model can discover the capability.
|
||||
return self._failure(
|
||||
f"{exc}. If you need a capability that is not "
|
||||
f"exposed, call {FIND_CAPABILITY_TOOL} with a "
|
||||
"description of what you need to do.",
|
||||
events,
|
||||
thinking,
|
||||
iterations_used=iterations_used,
|
||||
)
|
||||
raise
|
||||
requests = (
|
||||
normalize_ollama_tool_calls(cast(dict, assistant))
|
||||
if native_tools
|
||||
else normalize_text_tool_calls(assistant.get("content", ""))
|
||||
)
|
||||
iterations_used += 1
|
||||
except ValueError as exc:
|
||||
return self._failure(str(exc), events, thinking, iterations_used=iterations_used)
|
||||
@@ -669,31 +586,6 @@ class LocalToolRuntime:
|
||||
conversation.append(assistant)
|
||||
requests = self._deduplicate(requests)
|
||||
|
||||
def _expose_tools(self, names: "list[str] | tuple[str, ...] | set[str] | object") -> None:
|
||||
"""Add tool schemas by name for the rest of this run (P1-01 #24).
|
||||
|
||||
Only schemas that exist in the full registry can be exposed — a
|
||||
name without a schema is ignored. Already-exposed names are
|
||||
skipped; exposure is additive and never removes tools.
|
||||
"""
|
||||
added = False
|
||||
for name in (names if isinstance(names, (list, tuple, set, frozenset)) else []):
|
||||
if not isinstance(name, str) or name in self.tool_names:
|
||||
continue
|
||||
definition = TOOL_INDEX.get(name)
|
||||
if definition is None:
|
||||
continue
|
||||
self.tools.append(copy.deepcopy(definition))
|
||||
self.tool_names = frozenset(self.tool_names | {name})
|
||||
added = True
|
||||
if added:
|
||||
exposed_now = [
|
||||
name
|
||||
for name in (names if isinstance(names, (list, tuple, set, frozenset)) else [])
|
||||
if name in self.tool_names
|
||||
]
|
||||
LOG.info("capability escape hatch exposed tools in-run: %s", sorted(exposed_now))
|
||||
|
||||
@staticmethod
|
||||
def _deduplicate(requests: list[ToolRequest]) -> list[ToolRequest]:
|
||||
unique: list[ToolRequest] = []
|
||||
|
||||
@@ -43,7 +43,7 @@ def scripts_dir():
|
||||
"""Return the scripts directory."""
|
||||
return expand(os.environ.get(
|
||||
"RUNE_SCRIPTS_DIR",
|
||||
os.path.dirname(os.path.abspath(__file__))))
|
||||
os.path.join(rune_home(), "src", "rune")))
|
||||
|
||||
|
||||
def store_db():
|
||||
|
||||
@@ -0,0 +1,511 @@
|
||||
#!/usr/bin/env python3
|
||||
"""memory_selector.py — the ONE memory-selection API (P1-03, #26).
|
||||
|
||||
Authority map (config/v1_authority_map.json, domain ``memory_selection``):
|
||||
``ContextBroker.memory_selector`` is the v1_owner. Everything that surfaces
|
||||
memory into model context flows through ``MemorySelector.select_for_task`` —
|
||||
the operator registers exactly ONE memory contributor consuming it; the
|
||||
pre-P1-03 parallel contributors (recall, persona_memories, consolidated,
|
||||
neurons, graphs, abstraction) are replaced, not added to.
|
||||
|
||||
Policy (issue #26):
|
||||
- ONE relevance floor — ``memory_store.RECALL_MIN_SCORE`` (0.42) default —
|
||||
applied to EVERY source. The store source applies it inside the store's
|
||||
calibrated gate (``min_score`` is passed down into ``recall``, which
|
||||
enforces it in EVERY mode: lexical-only, dense-only, hybrid); self-
|
||||
scored sources are floored here. ``gate_consolidated``'s flat-0.8
|
||||
shortcut inside ``cached_recall`` is removed (Task 3) so consolidated
|
||||
knowledge competes under the same floor, and the top-5-importance neuron
|
||||
fallback is gone.
|
||||
- TRUE ABSTENTION: a trivial task legitimately receives ZERO recalled
|
||||
memories. Below-floor candidates are counted (the irrelevant-memory rate
|
||||
is computable per run) but never selected.
|
||||
- Memory-type targeting via ``memory_store``'s typed recall filter.
|
||||
- FAIL-CLOSED: a source failure is a skip + log, not an injection; if the
|
||||
selector itself is unavailable the contributor abstains — never fail open
|
||||
to "inject something important".
|
||||
|
||||
Degrade-not-crash throughout: sources are pluggable and isolated; one
|
||||
exploding source never takes down assembly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
try:
|
||||
from . import memory_store
|
||||
except ImportError: # direct script execution
|
||||
import memory_store # type: ignore[no-redef]
|
||||
|
||||
try:
|
||||
from .context_broker import ContributionBlock
|
||||
except ImportError: # direct script execution
|
||||
from context_broker import ContributionBlock # type: ignore[no-redef]
|
||||
|
||||
LOG = logging.getLogger("rune.memory_selector")
|
||||
|
||||
# Candidate cap BEFORE budgeting (plan DD 5: the selector caps its own
|
||||
# candidate list before returning).
|
||||
MAX_CANDIDATES = 24
|
||||
|
||||
# Default per-task candidate budget (the old memory contributor recalled 3;
|
||||
# the collapsed selector merges all sources under one slightly larger cap).
|
||||
DEFAULT_BUDGET = 4
|
||||
|
||||
# Priority of the single memory block (was _contributor_memory's 70).
|
||||
MEMORY_BLOCK_PRIORITY = 70
|
||||
|
||||
|
||||
@dataclass
|
||||
class SelectionResult:
|
||||
"""One selection outcome — mirror of CapabilityBroker's select_for_job.
|
||||
|
||||
``irrelevant_rate`` is computable per run:
|
||||
``(candidates_evaluated - selected_count) / candidates_evaluated`` when
|
||||
candidates > 0; abstained runs are recorded as zero-injection events.
|
||||
"""
|
||||
|
||||
blocks: list[ContributionBlock]
|
||||
candidates_evaluated: int
|
||||
selected_count: int
|
||||
abstained: bool
|
||||
types_queried: list[str] = field(default_factory=list)
|
||||
|
||||
def irrelevant_rate(self) -> float:
|
||||
if self.candidates_evaluated <= 0:
|
||||
return 0.0
|
||||
return (self.candidates_evaluated - self.selected_count) / self.candidates_evaluated
|
||||
|
||||
|
||||
class MemorySelector:
|
||||
"""Single memory-selection API composed by ContextBroker.
|
||||
|
||||
Sources are registered callables ``(query, types, budget, min_score) ->
|
||||
list[dict]`` where each dict carries at least ``text`` and ``score``
|
||||
(0-1 relevance under the ONE floor). The store source (``_store_recall``)
|
||||
routes through ``memory_store.recall``'s calibrated gate; Task 3 adds the
|
||||
remaining sources. A source failure is skipped + logged (fail-closed).
|
||||
|
||||
KNOWN RANKING BIAS (P1-03 #26 discharge, finding 9): the ONE floor gates
|
||||
each source on its OWN calibrated scale (lexical store ~4-6, dense/
|
||||
hybrid store RRF ~0-0.13 with the cosine gated at 0.42, self-scored
|
||||
0-1), but merged candidates are RANKED by raw score — so a lexical-mode
|
||||
store row outranks an equally-relevant self-scored row. Abstention is
|
||||
unaffected (every source is floored on its own scale); only candidate
|
||||
ORDER is biased. Per-source rank normalization was deliberately deferred:
|
||||
renormalizing the store's calibrated scales risked false abstentions
|
||||
this late in the release.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str | None = None, min_score: float | None = None):
|
||||
self.db_path = db_path if db_path is not None else memory_store.DB_PATH
|
||||
self.min_score = float(min_score) if min_score is not None else float(memory_store.RECALL_MIN_SCORE)
|
||||
# P1-03 (#26) DISCHARGE (finding 3): the selector's PINNED db_path is
|
||||
# bound into store-backed sources at registration time — a bare
|
||||
# ``memory_store.connect()`` inside a source used to open the
|
||||
# module-default store, ignoring the pin. Only sources that actually
|
||||
# declare a ``db_path`` parameter are bound (replacement sources
|
||||
# registered via add_source keep the documented 4-arg contract).
|
||||
self._sources: dict[str, Callable] = {}
|
||||
for name, fn in DEFAULT_SOURCES.items():
|
||||
try:
|
||||
params = inspect.signature(fn).parameters
|
||||
has_db_path = "db_path" in params or any(
|
||||
p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
has_db_path = False
|
||||
if has_db_path:
|
||||
self._sources[name] = functools.partial(fn, db_path=self.db_path)
|
||||
else:
|
||||
self._sources[name] = fn
|
||||
|
||||
# -- source registry -----------------------------------------------------
|
||||
|
||||
def add_source(self, name: str, fn: Callable) -> None:
|
||||
"""Register a selection source. Registration ORDER is execution order
|
||||
(dict preserves it); re-registering a name replaces that source."""
|
||||
self._sources[name] = fn
|
||||
|
||||
def source_names(self) -> list[str]:
|
||||
"""Registered source names in execution order."""
|
||||
return list(self._sources)
|
||||
|
||||
# -- selection -------------------------------------------------------------
|
||||
|
||||
def select_for_task(
|
||||
self,
|
||||
task_description: str,
|
||||
types: list[str] | None = None,
|
||||
budget: int = DEFAULT_BUDGET,
|
||||
) -> SelectionResult:
|
||||
"""Select memory for one task under the ONE floor.
|
||||
|
||||
Returns a SelectionResult; on total selector failure the caller
|
||||
(operator contributor) treats any exception as an abstention.
|
||||
"""
|
||||
query = (task_description or "").strip()
|
||||
types_queried = list(types) if types else []
|
||||
if not query:
|
||||
return SelectionResult([], 0, 0, True, types_queried)
|
||||
|
||||
candidates: list[dict] = []
|
||||
for name, fn in self._sources.items():
|
||||
try:
|
||||
found = fn(query, types_queried, MAX_CANDIDATES, self.min_score)
|
||||
except Exception as exc: # fail-closed per source
|
||||
LOG.warning("memory source %r failed; skipping (%s)", name, exc)
|
||||
continue
|
||||
for item in found or []:
|
||||
entry = dict(item)
|
||||
entry["kind"] = entry.get("kind") or name
|
||||
candidates.append(entry)
|
||||
candidates = candidates[:MAX_CANDIDATES]
|
||||
|
||||
# ONE floor for EVERY source: the store source was already floored by
|
||||
# the same constant inside recall() — which applies min_score in
|
||||
# EVERY mode (lexical-only, dense-only, hybrid; P1-03 #26 discharge)
|
||||
# — so its rows are tagged store_scored and exempt from this
|
||||
# selector-side numeric check. Self-scored sources are floored here
|
||||
# against the same selector min_score.
|
||||
selected = [item for item in candidates if item.get("store_scored") or _score_of(item) >= self.min_score]
|
||||
selected.sort(key=lambda item: -_score_of(item))
|
||||
selected = selected[: max(0, budget)]
|
||||
|
||||
blocks: list[ContributionBlock] = []
|
||||
for item in selected:
|
||||
text = str(item.get("text", "")).strip()
|
||||
if not text:
|
||||
continue
|
||||
blocks.append(
|
||||
ContributionBlock(
|
||||
"memory",
|
||||
MEMORY_BLOCK_PRIORITY,
|
||||
f"- {text}",
|
||||
max(1, len(text) // 4),
|
||||
"memory_selector.select_for_task",
|
||||
evictable=True,
|
||||
)
|
||||
)
|
||||
|
||||
abstained = not blocks
|
||||
if abstained:
|
||||
LOG.info(
|
||||
"memory abstain: task=%r candidates=%d floor=%.2f types=%s",
|
||||
query[:60],
|
||||
len(candidates),
|
||||
self.min_score,
|
||||
types_queried or "all",
|
||||
)
|
||||
return SelectionResult(blocks, len(candidates), len(blocks), abstained, types_queried)
|
||||
|
||||
# -- sources ---------------------------------------------------------------
|
||||
|
||||
def _source_store(self, query: str, types: list[str], budget: int, min_score: float) -> list[dict]:
|
||||
"""Typed store recall through the ONE thresholded path."""
|
||||
return _store_recall(query, types, budget, min_score, db_path=self.db_path)
|
||||
|
||||
|
||||
def _score_of(item: dict) -> float:
|
||||
"""Extract the item's 0-1 relevance under the ONE floor. Missing or
|
||||
malformed scores floor at 0.0 (abstain) — never guessed upward."""
|
||||
value = item.get("score")
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
return 0.0
|
||||
return float(value)
|
||||
|
||||
|
||||
def _store_recall(
|
||||
query: str,
|
||||
types: list[str] | None,
|
||||
budget: int,
|
||||
min_score: float = memory_store.RECALL_MIN_SCORE,
|
||||
db_path: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Store source: typed recall through the ONE thresholded path.
|
||||
|
||||
Uses ``memory_store.recall`` (the calibrated QPP gate) — never the
|
||||
``cached_recall`` path whose ``gate_consolidated`` flat-0.8 shortcut
|
||||
bypasses thresholding (removed in Task 3).
|
||||
|
||||
The store's ``relevance`` lives on mode-dependent calibrated scales
|
||||
(lexical-mode ~4-6 via the +5 offset band, dense/hybrid-mode RRF ~0-0.13,
|
||||
with the dense COSINE gated at 0.42-0.8 — see memory_store.recall), so it
|
||||
is NOT renormalized here: the store already applied THE floor via
|
||||
``min_score`` in EVERY recall mode (lexical-only, dense-only, hybrid —
|
||||
P1-03 #26 discharge), so tagging the result ``store_scored=True`` and
|
||||
exempting it from the selector's numeric check is sound. The result is
|
||||
tagged ``store_scored=True`` so the selector treats these rows as
|
||||
floor-cleared by the same constant.
|
||||
|
||||
KNOWN RANKING BIAS (P1-03 #26 discharge, finding 9): because the scales
|
||||
differ per mode, the selector's raw-score ranking favors lexical-mode
|
||||
store rows (~4-6) over self-scored rows (0-1) and dense/hybrid-mode
|
||||
store rows (~0-0.13). Abstention is NOT affected (each source is floored
|
||||
on its own calibrated scale); only the ORDER of merged candidates is
|
||||
biased. Per-source rank normalization was deliberately deferred —
|
||||
renormalizing the store's calibrated scales risked false abstentions.
|
||||
|
||||
P1-03 (#26) DISCHARGE (finding 2a): besides the caller-typed query, the
|
||||
source always runs a SECOND typed query restricted to
|
||||
``self``/``personality`` — the persona lane. The production caller (the
|
||||
operator's single memory contributor) invokes the selector WITHOUT
|
||||
types, which left the typed persona source dead in production. Both
|
||||
queries are thresholded inside the store's gate and may abstain — a
|
||||
trivial task still gets zero persona rows. Results are merged and
|
||||
capped at ``budget``.
|
||||
|
||||
P1-03 (#26) DEVIATION: shard routing inside ``recall`` is bypassed — the
|
||||
selector pins the canonical store connection. ShardManager routes to a
|
||||
hardcoded ~/.config/opencode shard dir that ignores RUNE_* isolation and
|
||||
carries no memory_type column, so routing through it would break typed
|
||||
recall and the preserve-stores/isolation invariants.
|
||||
"""
|
||||
db = memory_store.connect(db_path)
|
||||
try:
|
||||
results = memory_store.recall(db, query, budget=budget, min_score=min_score, memory_types=types or None)
|
||||
persona_types = ["self", "personality"]
|
||||
if not (set(types or []) & set(persona_types)):
|
||||
# The persona lane: always executed for the untyped production
|
||||
# caller (and any caller that did not itself ask persona types).
|
||||
results = results + memory_store.recall(
|
||||
db, query, budget=budget, min_score=min_score, memory_types=persona_types
|
||||
)
|
||||
finally:
|
||||
db.close()
|
||||
out: list[dict] = []
|
||||
seen_ids: set = set()
|
||||
for row in results:
|
||||
# The untyped and typed-lane queries overlap on unclassified rows —
|
||||
# merge by entry id so a shared row is a candidate ONCE (duplicates
|
||||
# would corrupt the irrelevant-memory rate and repeat block text).
|
||||
row_id = row.get("id")
|
||||
if row_id is not None:
|
||||
if row_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(row_id)
|
||||
out.append(
|
||||
{
|
||||
"text": row.get("text", ""),
|
||||
"score": float(row.get("relevance", 0.0)),
|
||||
"store_scored": True,
|
||||
"kind": "store",
|
||||
"id": row.get("id"),
|
||||
"source": row.get("source", ""),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _lex_overlap_score(text: str, query: str) -> float:
|
||||
"""Deterministic lexical relevance for self-scored sources.
|
||||
|
||||
Distinctive-word coverage of the QUERY terms present in the candidate
|
||||
text, with a stopword filter shared with the store's gate vocabulary.
|
||||
A candidate sharing NO distinctive query word cannot score — the same
|
||||
conjunctive logic the store's BM25 gate applies (a shared single word is
|
||||
not a relation, but zero shared words is definitely not one).
|
||||
"""
|
||||
stopwords = getattr(memory_store, "_STOPWORDS", set())
|
||||
q_terms = {t.lower() for t in query.split() if len(t) > 3 and t.lower() not in stopwords}
|
||||
if not q_terms:
|
||||
return 0.0
|
||||
text_lower = (text or "").lower()
|
||||
hits = sum(1 for term in q_terms if term in text_lower)
|
||||
return hits / len(q_terms)
|
||||
|
||||
|
||||
def _neurons_recall(query: str, types: list[str] | None, budget: int, min_score: float) -> list[dict]:
|
||||
"""Warm-neuron source: trigger scan, thresholded — NO importance fallback.
|
||||
|
||||
The pre-P1-03 contributor injected the top-5 neurons by importance when
|
||||
no trigger matched (rune_operator.py:1387-1389). That fallback is dead:
|
||||
a neuron that does not match the task is not injected merely because it
|
||||
is important. Trigger-matched neurons are scored lexically against the
|
||||
query and must clear the same floor as every other source.
|
||||
"""
|
||||
try:
|
||||
import warm_neuron_store
|
||||
except ImportError: # TUI top-level deploy
|
||||
import rune.warm_neuron_store as warm_neuron_store # type: ignore[no-redef]
|
||||
|
||||
neurons = warm_neuron_store.list_neurons()
|
||||
task_words = {w.lower().strip(".,;:!?()[]{}\"'") for w in query.split()}
|
||||
out: list[dict] = []
|
||||
for neuron in neurons:
|
||||
triggers = neuron.get("triggers") or []
|
||||
if not any(t in task_words for t in triggers):
|
||||
continue
|
||||
text = neuron.get("text", "")
|
||||
score = _lex_overlap_score(text, query)
|
||||
if score < min_score:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"text": f"[{neuron.get('slug', 'warm')}] {text}",
|
||||
"score": score,
|
||||
"kind": "neuron",
|
||||
"source": "warm_neuron_store.list_neurons",
|
||||
}
|
||||
)
|
||||
if len(out) >= budget:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _graphs_recall(query: str, types: list[str] | None, budget: int, min_score: float) -> list[dict]:
|
||||
"""Graph source: entity BFS from the task words via graph_memory.query,
|
||||
floored. The pre-P1-03 contributor called resolve_entity(), which MUTATES
|
||||
the graph (creates missing entities); read-only query() keeps the store
|
||||
side-effect-free. A term with no entity yields no candidates."""
|
||||
try:
|
||||
import graph_memory
|
||||
except ImportError: # TUI top-level deploy
|
||||
import rune.graph_memory as graph_memory # type: ignore[no-redef]
|
||||
|
||||
db = graph_memory.connect()
|
||||
out: list[dict] = []
|
||||
try:
|
||||
task_words = list(dict.fromkeys(w.lower().strip(".,;:!?()[]{}\"'") for w in query.split()))[:5]
|
||||
for word in task_words:
|
||||
result = graph_memory.query(db, word, hops=1)
|
||||
if not result:
|
||||
continue
|
||||
entity = result.get("entity", word) if isinstance(result, dict) else str(result)
|
||||
facts = result.get("facts", []) if isinstance(result, dict) else []
|
||||
lines = [f"{f.get('subject')} {f.get('predicate', '-')} {f.get('object')}" for f in facts]
|
||||
text = "; ".join(lines) if lines else str(entity)
|
||||
score = _lex_overlap_score(text, query) or (1.0 / max(len(task_words), 1))
|
||||
if score < min_score:
|
||||
continue
|
||||
out.append({"text": text, "score": score, "kind": "graph", "source": "graph_memory.query"})
|
||||
if len(out) >= budget:
|
||||
break
|
||||
finally:
|
||||
db.close()
|
||||
return out
|
||||
|
||||
|
||||
def _abstraction_recall(query: str, types: list[str] | None, budget: int, min_score: float) -> list[dict]:
|
||||
"""Abstraction-facts source: LIKE scan over active facts, floored.
|
||||
|
||||
The pre-P1-03 contributor injected the top-3 LIKE matches unconditionally;
|
||||
here a LIKE match is only a candidate — it must clear the floor.
|
||||
"""
|
||||
try:
|
||||
import abstraction
|
||||
except ImportError: # TUI top-level deploy
|
||||
import rune.abstraction as abstraction # type: ignore[no-redef]
|
||||
|
||||
db = abstraction.connect()
|
||||
out: list[dict] = []
|
||||
try:
|
||||
task_words = [w for w in query.lower().split()[:3] if w]
|
||||
if not task_words:
|
||||
return []
|
||||
placeholders = " OR ".join(["what LIKE ?" for _ in task_words])
|
||||
params = [f"%{w}%" for w in task_words]
|
||||
rows = db.execute(
|
||||
f"SELECT what FROM facts WHERE state='active' AND ({placeholders}) LIMIT {int(budget) * 3}",
|
||||
params,
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
text = str(row[0])
|
||||
score = _lex_overlap_score(text, query)
|
||||
if score < min_score:
|
||||
continue
|
||||
out.append({"text": text, "score": score, "kind": "abstraction", "source": "abstraction facts"})
|
||||
if len(out) >= budget:
|
||||
break
|
||||
finally:
|
||||
db.close()
|
||||
return out
|
||||
|
||||
|
||||
def _consolidated_recall(query: str, types: list[str] | None, budget: int, min_score: float) -> list[dict]:
|
||||
"""Consolidated-knowledge source under the SAME floor (bypass removed).
|
||||
|
||||
The pre-P1-03 path handed ``gate_consolidated``'s output a flat
|
||||
relevance=0.8, bypassing thresholding entirely. Consolidated knowledge
|
||||
now competes like any candidate: EACH knowledge item is scored by its
|
||||
lexical overlap with the active task, and below-floor items abstain.
|
||||
"""
|
||||
try:
|
||||
from .memory_consolidation import MemoryConsolidation
|
||||
except ImportError: # TUI top-level deploy
|
||||
from memory_consolidation import MemoryConsolidation # type: ignore[no-redef]
|
||||
|
||||
try:
|
||||
consolidation = MemoryConsolidation()
|
||||
items = list(consolidation.get_relevant_knowledge(query, limit=budget * 2))
|
||||
except Exception as exc: # degrade: consolidation unavailable -> no candidates
|
||||
LOG.warning("consolidation source unavailable; skipping (%s)", exc)
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for item in items:
|
||||
content = (getattr(item, "content", "") or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
score = _lex_overlap_score(content, query)
|
||||
if score < min_score:
|
||||
continue # below floor -> abstain (was: flat relevance injection)
|
||||
out.append({"text": content, "score": score, "kind": "consolidated", "source": "memory_consolidation"})
|
||||
if len(out) >= budget:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _persona_recall(
|
||||
query: str,
|
||||
types: list[str] | None,
|
||||
budget: int,
|
||||
min_score: float,
|
||||
db_path: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""Typed persona source: self/personality memory keyed on the ACTIVE task.
|
||||
|
||||
Replaces the hardcoded ``"Rune personality voice identity how to speak"``
|
||||
query. Only runs when the caller asked for self/personality types (the
|
||||
operator's persona-memory call site); thresholded, may abstain — a
|
||||
trivial task gets zero persona lines.
|
||||
|
||||
P1-03 (#26) DISCHARGE (finding 3): the source queries the selector's
|
||||
PINNED store (``db_path``) — never the module-default store that a bare
|
||||
``memory_store.connect()`` would open.
|
||||
"""
|
||||
wanted = {t for t in (types or []) if t in ("self", "personality")}
|
||||
if not wanted:
|
||||
return []
|
||||
db = memory_store.connect(db_path)
|
||||
try:
|
||||
rows = memory_store.recall(db, query, budget=budget, min_score=min_score, memory_types=sorted(wanted))
|
||||
finally:
|
||||
db.close()
|
||||
out: list[dict] = []
|
||||
for row in rows:
|
||||
out.append(
|
||||
{
|
||||
"text": row.get("text", ""),
|
||||
"score": float(row.get("relevance", 0.0)),
|
||||
"kind": "persona",
|
||||
"source": "memory_store.recall(self/personality)",
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# The default source set (P1-03 #26): every pre-selector memory path, unified
|
||||
# under the ONE floor. Registration order = execution order.
|
||||
DEFAULT_SOURCES: dict[str, Callable] = {
|
||||
"store": _store_recall,
|
||||
"neurons": _neurons_recall,
|
||||
"graphs": _graphs_recall,
|
||||
"abstraction": _abstraction_recall,
|
||||
"consolidated": _consolidated_recall,
|
||||
"persona": _persona_recall,
|
||||
}
|
||||
@@ -470,7 +470,7 @@ def add_entry(db, text, importance=0.5, topic="", entities=None,
|
||||
return True
|
||||
|
||||
|
||||
def recall(db, query, budget=RECALL_BUDGET, min_score=RECALL_MIN_SCORE):
|
||||
def recall(db, query, budget=RECALL_BUDGET, min_score=RECALL_MIN_SCORE, *, probe_cap: int = RECALL_BUDGET * 2):
|
||||
"""Hybrid recall: dense + BM25 fused by RRF, metadata-aware.
|
||||
|
||||
Returns (entries, scores) — the top `budget` active entries. If nothing
|
||||
@@ -486,7 +486,17 @@ def recall(db, query, budget=RECALL_BUDGET, min_score=RECALL_MIN_SCORE):
|
||||
|
||||
CACHE LAYER (2026-08-21): check cache before expensive retrieval.
|
||||
Cache is invalidated by event bus when memories change.
|
||||
|
||||
M8 (#161): the internal dense/FTS probe caps are parameterized via
|
||||
`probe_cap` (default RECALL_BUDGET*2 — byte-identical behavior when
|
||||
unset). The benchmark harness sweeps this alongside top-k to expose
|
||||
candidate-pool starvation vs rank-ordering failures (INV-5).
|
||||
|
||||
M1 (#161): every returned entry carries `mode` — 'lexical-only',
|
||||
'dense-only', or 'hybrid' — recording which fusion branch produced it.
|
||||
Observability ONLY (INV-13): no consumer may branch behavior on it.
|
||||
"""
|
||||
|
||||
query = (query or "").strip()
|
||||
if not query:
|
||||
return []
|
||||
@@ -524,7 +534,7 @@ def recall(db, query, budget=RECALL_BUDGET, min_score=RECALL_MIN_SCORE):
|
||||
rows = db.execute(
|
||||
"SELECT v.rowid, distance FROM entries_vec v "
|
||||
"WHERE v.embedding MATCH ? AND k = ?",
|
||||
(json.dumps(qvec), RECALL_BUDGET * 2)).fetchall()
|
||||
(json.dumps(qvec), probe_cap)).fetchall()
|
||||
# META-JUDGEMENT (applies in BOTH modes): vector proximity is not a
|
||||
# relation by itself. A dense hit must also share a CONTENT WORD with
|
||||
# the query — otherwise "black coffee" would pull in "black holes"
|
||||
@@ -574,7 +584,7 @@ def recall(db, query, budget=RECALL_BUDGET, min_score=RECALL_MIN_SCORE):
|
||||
"SELECT rowid, bm25(entries_fts) AS score FROM entries_fts "
|
||||
"WHERE entries_fts MATCH ? "
|
||||
"ORDER BY bm25(entries_fts) LIMIT ?",
|
||||
(_fts_query_conjunctive(query), RECALL_BUDGET * 2)).fetchall()
|
||||
(_fts_query_conjunctive(query), probe_cap)).fetchall()
|
||||
# Keep REAL BM25 scores (Robertson & Zaragoza 2009) for the QPP
|
||||
# threshold — never flatten them.
|
||||
bm25 = [(r["rowid"], float(r["score"])) for r in rows]
|
||||
@@ -600,7 +610,7 @@ def recall(db, query, budget=RECALL_BUDGET, min_score=RECALL_MIN_SCORE):
|
||||
except Exception:
|
||||
pass
|
||||
if _expanded_hits:
|
||||
_ids = list(_expanded_hits)[:RECALL_BUDGET * 2]
|
||||
_ids = list(_expanded_hits)[:probe_cap]
|
||||
_ph = ",".join("?" * len(_ids))
|
||||
_rows = db.execute(
|
||||
f"SELECT rowid, bm25(entries_fts) AS score FROM entries_fts "
|
||||
@@ -614,13 +624,19 @@ def recall(db, query, budget=RECALL_BUDGET, min_score=RECALL_MIN_SCORE):
|
||||
# arXiv:2210.11934) — use raw BM25 scores directly.
|
||||
ranks = {}
|
||||
best_dense = 0.0
|
||||
# M1 observability (#161 synthesis INV-13): which fusion branch produced
|
||||
# the results. Observability ONLY — consumers must never branch behavior
|
||||
# on this field (that would create a second retrieval authority).
|
||||
retrieval_mode = "lexical-only"
|
||||
if dense and bm25:
|
||||
retrieval_mode = "hybrid"
|
||||
for idx, lst in enumerate([dense, bm25]):
|
||||
for pos, (eid, _score) in enumerate(lst):
|
||||
ranks[eid] = ranks.get(eid, 0.0) + 1.0 / (RRF_K + pos + 1)
|
||||
if idx == 0:
|
||||
best_dense = max(best_dense, _score)
|
||||
elif dense:
|
||||
retrieval_mode = "dense-only"
|
||||
for pos, (eid, _score) in enumerate(dense):
|
||||
ranks[eid] = 1.0 / (RRF_K + pos + 1)
|
||||
best_dense = max(best_dense, _score)
|
||||
@@ -687,6 +703,9 @@ def recall(db, query, budget=RECALL_BUDGET, min_score=RECALL_MIN_SCORE):
|
||||
"confidence": round(conf, 2),
|
||||
"temperature": round(temp, 2),
|
||||
"relevance": round(relevance, 4),
|
||||
# M1 observability (#161): lexical-only | dense-only | hybrid.
|
||||
# NEVER consumed for behavior — see INV-13 in the synthesis.
|
||||
"mode": retrieval_mode,
|
||||
})
|
||||
# Enforce budget + abstention floor.
|
||||
if not scored:
|
||||
|
||||
@@ -1,432 +0,0 @@
|
||||
"""Model/provider routing — one authoritative router (#29, P1-06).
|
||||
|
||||
Owns the ``ProviderAdapter`` protocol (forge-Protocol idiom), the first
|
||||
live adapter (Ollama, delegating to ``OllamaChatTransport``), the measured
|
||||
registry rows (``MeasuredModelProfile``), and ``ModelRouter.rank()`` — an
|
||||
advisory ranked-candidate selector in the ``orchestration/router.py``
|
||||
idiom: it never mutates its inputs and never executes; the operator tier
|
||||
loop remains the final decider.
|
||||
|
||||
Authority-map migration rule (model_routing): "One router selects the
|
||||
cheapest verified-capable model from measured profiles and records the
|
||||
reason."
|
||||
|
||||
Design decisions (settled, plan v1):
|
||||
- "Verified-capable" = measured success rate per task family from
|
||||
``inference_traces.jsonl`` + static capability floor (tier ordering,
|
||||
``supports_tools``, ``context_window``). No OutcomeLedger dependency
|
||||
yet (#33 adopts later).
|
||||
- Escalation is a START-TIER decision for the NEXT run keyed off the
|
||||
run-level result — never a mid-run cloud burn.
|
||||
- Cloud adapters are declared, not live: ``CloudStubAdapter.chat``
|
||||
raises ``ProviderUnavailable`` (never silently fake).
|
||||
- Measured profiles persist to ``<rune_home>/state/model_routing_profiles.json``
|
||||
keyed by ``(provider, model)``; disconnecting a provider removes only
|
||||
the adapter instance — profiles and outcomes stay on disk.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
class ProviderUnavailable(RuntimeError):
|
||||
"""A declared provider has no live credentials/endpoint configured."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ProviderAdapter(Protocol):
|
||||
"""Chat-only provider boundary (forge-Protocol idiom, forge_client.py).
|
||||
|
||||
``chat`` returns a ChatResponse-shaped dict (Ollama ``/api/chat``
|
||||
envelope shape: ``{"message": {...}, ...}``). Capability discovery
|
||||
stays in ``local_tool_runtime`` (#28 boundary).
|
||||
"""
|
||||
|
||||
name: str
|
||||
|
||||
def available(self) -> bool: ...
|
||||
|
||||
def chat(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
options: dict | None = None,
|
||||
think: bool | str | None = None,
|
||||
) -> dict: ...
|
||||
|
||||
|
||||
class OllamaAdapter:
|
||||
"""First live adapter — wraps ``OllamaChatTransport``."""
|
||||
|
||||
name = "ollama"
|
||||
|
||||
#: /api/tags ping result cache window (seconds).
|
||||
_AVAILABILITY_TTL = 30.0
|
||||
|
||||
def __init__(self, transport=None, base_url: str = "http://localhost:11434"):
|
||||
if transport is None:
|
||||
from rune.local_tool_runtime import OllamaChatTransport
|
||||
|
||||
transport = OllamaChatTransport(base_url=base_url)
|
||||
self._transport = transport
|
||||
self._available_cache: tuple[bool, float] | None = None
|
||||
|
||||
def available(self) -> bool:
|
||||
"""Ping ``/api/tags`` with a 2s timeout; cached for 30s.
|
||||
|
||||
Transports that carry their own availability probe (test doubles,
|
||||
future multi-endpoint transports) are consulted directly.
|
||||
"""
|
||||
transport_probe = getattr(self._transport, "available", None)
|
||||
if callable(transport_probe) and not hasattr(self._transport, "base_url"):
|
||||
return bool(transport_probe())
|
||||
cached = self._available_cache
|
||||
if cached is not None and (time.monotonic() - cached[1]) < self._AVAILABILITY_TTL:
|
||||
return cached[0]
|
||||
ok = False
|
||||
try:
|
||||
base = getattr(self._transport, "base_url", "http://localhost:11434")
|
||||
try:
|
||||
import httpx
|
||||
|
||||
response = httpx.get(f"{base.rstrip('/')}/api/tags", timeout=2.0)
|
||||
ok = response.status_code == 200
|
||||
except ImportError:
|
||||
import urllib.request
|
||||
|
||||
with urllib.request.urlopen(f"{base.rstrip('/')}/api/tags", timeout=2.0) as resp:
|
||||
ok = resp.status == 200
|
||||
except Exception:
|
||||
ok = False
|
||||
self._available_cache = (ok, time.monotonic())
|
||||
return ok
|
||||
|
||||
def chat(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
options: dict | None = None,
|
||||
think: bool | str | None = None,
|
||||
) -> dict:
|
||||
"""Delegate to the wrapped transport; normalize to a dict envelope.
|
||||
|
||||
The transport returns a ``ChatMessage`` (the Ollama ``message``
|
||||
object). The adapter lifts it into the response-envelope shape so
|
||||
callers see one consistent contract.
|
||||
"""
|
||||
message = self._transport.chat(model=model, messages=messages, tools=tools, options=options, think=think)
|
||||
if isinstance(message, dict) and "message" in message:
|
||||
return message
|
||||
return {"message": dict(message)}
|
||||
|
||||
|
||||
class CloudStubAdapter:
|
||||
"""Declared cloud adapter — NOT live (no credentials enter #29).
|
||||
|
||||
Keeping the interface honest: a declared-but-unconfigured provider
|
||||
fails loudly instead of silently faking a chat response.
|
||||
"""
|
||||
|
||||
name = "cloud"
|
||||
|
||||
def available(self) -> bool:
|
||||
return False
|
||||
|
||||
def chat(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
options: dict | None = None,
|
||||
think: bool | str | None = None,
|
||||
) -> dict:
|
||||
raise ProviderUnavailable("cloud adapter declared but not configured (no credentials in #29)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Measured registry (Task 2 substrate)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MeasuredModelProfile:
|
||||
"""Per ``(provider, model)`` measured aggregates from inference traces.
|
||||
|
||||
Built from ``inference_traces.jsonl`` rows (already persisted,
|
||||
append-only). ``per_family_success`` maps task_family →
|
||||
(successes, attempts).
|
||||
"""
|
||||
|
||||
provider: str
|
||||
model: str
|
||||
per_family_success: dict = field(default_factory=dict) # family -> [ok, n]
|
||||
avg_latency_ms: float = 0.0
|
||||
estimated_cost_usd: float = 0.0
|
||||
|
||||
|
||||
def load_profiles(path) -> dict[tuple[str, str], MeasuredModelProfile]:
|
||||
"""Load measured profiles from ``state/model_routing_profiles.json``.
|
||||
|
||||
The only I/O helper in this module — ``rank()`` stays pure. Missing
|
||||
file → empty registry (capability-floor ordering applies).
|
||||
"""
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
rows = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
if not isinstance(rows, list):
|
||||
return {}
|
||||
out: dict[tuple[str, str], MeasuredModelProfile] = {}
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
key = (str(row.get("provider", "")), str(row.get("model", "")))
|
||||
# quality I-5: coerce/drop non-numeric per-family stats at load time
|
||||
# so rank() never sees string garbage from a hand-edited JSON.
|
||||
raw_stats = row.get("per_family_success", {})
|
||||
clean_stats: dict[str, tuple[int, int]] = {}
|
||||
if isinstance(raw_stats, dict):
|
||||
for family, pair in raw_stats.items():
|
||||
if (
|
||||
isinstance(pair, (list, tuple))
|
||||
and len(pair) == 2
|
||||
and all(isinstance(v, int) and not isinstance(v, bool) for v in pair)
|
||||
):
|
||||
clean_stats[str(family)] = (pair[0], pair[1])
|
||||
if not clean_stats:
|
||||
# every family record was garbage → nothing measurable to keep
|
||||
continue
|
||||
try:
|
||||
avg_latency = float(row.get("avg_latency_ms", 0.0))
|
||||
est_cost = float(row.get("estimated_cost_usd", 0.0))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
out[key] = MeasuredModelProfile(
|
||||
provider=key[0],
|
||||
model=key[1],
|
||||
per_family_success=clean_stats,
|
||||
avg_latency_ms=avg_latency,
|
||||
estimated_cost_usd=est_cost,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def default_profiles_path() -> Path:
|
||||
"""Measured-profile store: ``<rune_home>/state/model_routing_profiles.json``.
|
||||
|
||||
Same durability pattern as model_profiles: keyed by provider+model on
|
||||
disk, independent of adapter instances — disconnecting a provider
|
||||
removes only the adapter object, never the measured state.
|
||||
"""
|
||||
try:
|
||||
from .memory_env import rune_home
|
||||
except ImportError: # direct script execution
|
||||
from rune.memory_env import rune_home
|
||||
|
||||
return Path(rune_home()) / "state" / "model_routing_profiles.json"
|
||||
|
||||
|
||||
def save_profiles(path, profiles: dict) -> Path:
|
||||
"""Persist measured profiles (list of dicts) to ``path``."""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
rows = []
|
||||
for (provider, model), profile in profiles.items():
|
||||
rows.append(
|
||||
{
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"per_family_success": profile.per_family_success,
|
||||
"avg_latency_ms": profile.avg_latency_ms,
|
||||
"estimated_cost_usd": profile.estimated_cost_usd,
|
||||
}
|
||||
)
|
||||
tmp = Path(str(path) + f".tmp{os.getpid()}")
|
||||
tmp.write_text(json.dumps(rows, indent=2), encoding="utf-8")
|
||||
# quality I-4: atomic — readers never observe a torn file
|
||||
os.replace(tmp, path)
|
||||
return path
|
||||
|
||||
|
||||
def update_profiles_from_traces(
|
||||
path, traces: list[dict], task_family: str = ""
|
||||
) -> dict[tuple[str, str], MeasuredModelProfile]:
|
||||
"""Fold inference-trace rows into the measured registry (Task 3 wiring).
|
||||
|
||||
``traces`` are dict-shaped trace rows (from ``read_traces`` or
|
||||
fixtures) with ``provider``, ``model``, ``success``, and optionally
|
||||
``task_family`` (overridden by the argument). Aggregation is pure —
|
||||
the caller owns I/O through ``load_profiles``/``save_profiles``.
|
||||
"""
|
||||
profiles = load_profiles(path)
|
||||
for row in traces:
|
||||
provider = str(row.get("provider", ""))
|
||||
model = str(row.get("model", ""))
|
||||
if not provider or not model:
|
||||
continue
|
||||
family = str(row.get("task_family", "") or task_family)
|
||||
if not family:
|
||||
continue
|
||||
key = (provider, model)
|
||||
profile = profiles.get(key) or MeasuredModelProfile(provider=provider, model=model)
|
||||
ok, n = profile.per_family_success.get(family, (0, 0))
|
||||
ok, n = int(ok), int(n)
|
||||
n += 1
|
||||
if row.get("success") is True:
|
||||
ok += 1
|
||||
profile.per_family_success[family] = (ok, n)
|
||||
profiles[key] = profile
|
||||
return profiles
|
||||
|
||||
|
||||
def build_model_router(profiles_path=None) -> ModelRouter:
|
||||
"""Single-authority router factory (composition wires this once)."""
|
||||
return ModelRouter(profiles_path=profiles_path or default_profiles_path())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ModelRouter (Task 2) — advisory ranked selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: Static capability floor (model_registry.ModelProfile tier ladder):
|
||||
#: tier id → (model id, provider, supports_tools, context_window, cost_usd).
|
||||
CAPABILITY_FLOOR = (
|
||||
# ("deterministic", "local", "T0", ...) REMOVED (quality discharge): the
|
||||
# T0 pseudo-model is not executable by any transport — ranking it made
|
||||
# the operator send the literal id "deterministic" to Ollama. T0 remains
|
||||
# a valid tier NUMBER in the ladder; no runnable T0 model exists yet.
|
||||
("qwen2.5:3b", "ollama", "T1", False, 32768, 0.0),
|
||||
("qwen3:14b", "ollama", "T2", False, 32768, 0.0),
|
||||
("qwen3.6:27b", "ollama", "T3", False, 32768, 0.0),
|
||||
("mimo-v2.5-pro", "openai", "T4", True, 128000, 15.0),
|
||||
)
|
||||
|
||||
#: Measured precedence needs at least this many family samples before a
|
||||
#: candidate may outrank the capability floor ordering (quality I-5).
|
||||
_MIN_MEASURED_SAMPLES = 2
|
||||
|
||||
|
||||
def _tier_number(tier: str) -> int:
|
||||
"""'T12' -> 12 (quality discharge: tier ids must sort numerically)."""
|
||||
try:
|
||||
return int(str(tier)[1:])
|
||||
except (TypeError, ValueError):
|
||||
return 10**6
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoutingDecision:
|
||||
"""One ranked candidate: model, provider, evidence reason, est. cost."""
|
||||
|
||||
model: str
|
||||
provider: str
|
||||
tier: str
|
||||
reason: str
|
||||
estimated_cost: float
|
||||
|
||||
|
||||
class ModelRouter:
|
||||
"""Advisory router: rank verified-capable candidates, cheapest first.
|
||||
|
||||
Mirrors ``orchestration/router.py.CostBiasRouter``: ``rank()`` is a
|
||||
pure function of (candidates, requirement) → ordered decisions. It
|
||||
never mutates input and never executes a model call.
|
||||
"""
|
||||
|
||||
policy_name = "model_routing"
|
||||
|
||||
def __init__(self, candidates=CAPABILITY_FLOOR, profiles_path=None, profiles=None):
|
||||
self._candidates = tuple(candidates)
|
||||
if profiles is not None:
|
||||
self._profiles = dict(profiles)
|
||||
elif profiles_path is not None:
|
||||
self._profiles = load_profiles(profiles_path)
|
||||
else:
|
||||
self._profiles = {}
|
||||
|
||||
def rank(self, task_family: str, need_tools: bool = False, min_context: int = 0) -> list:
|
||||
"""Rank eligible candidates: cheapest verified-capable first.
|
||||
|
||||
Eligibility: static capability floor (``supports_tools`` when
|
||||
tools are needed, ``context_window`` ≥ ``min_context``). Every
|
||||
decision carries its evidence reason:
|
||||
|
||||
- no measured profiles at all → ``"no-measured-profiles tier<N>"``
|
||||
(pure capability-floor ordering);
|
||||
- measured profiles exist → only verified-capable candidates
|
||||
(those with measured family stats and non-zero measured
|
||||
success) are ranked, with ``"measured <ok>/<n> success
|
||||
task_family=<family>"`` reasons.
|
||||
|
||||
Ordering: lowest estimated cost first (ties broken deterministically
|
||||
by tier, then model id) — cheapest verified-capable wins.
|
||||
"""
|
||||
decisions: list[RoutingDecision] = []
|
||||
have_profiles = bool(self._profiles)
|
||||
for model, provider, tier, supports_tools, context_window, cost in self._candidates:
|
||||
key = (provider, model)
|
||||
profile = self._profiles.get(key)
|
||||
stats = profile.per_family_success.get(task_family) if profile else None
|
||||
measured = None
|
||||
if stats is not None and len(stats) == 2:
|
||||
ok, n = stats
|
||||
# quality I-5: coerce non-numeric JSON garbage defensively
|
||||
if isinstance(ok, int) and isinstance(n, int):
|
||||
measured = (ok, n)
|
||||
if measured is not None and measured[1] > 0:
|
||||
ok, n = measured
|
||||
if ok == 0:
|
||||
# zero measured success in this family → not verified-capable
|
||||
continue
|
||||
# min_context is a PHYSICAL limit — measured success cannot
|
||||
# make a model hold more context than it has (quality I-5).
|
||||
if min_context > 0 and context_window < min_context:
|
||||
continue
|
||||
# need_tools: measured success in a tool-using family IS
|
||||
# tool-support evidence — overrides the static claim (the
|
||||
# frozen semantics; static floor only gates unmeasured).
|
||||
# measured verification (any n>0) makes the candidate
|
||||
# verified-capable; the reason carries the sample count so
|
||||
# callers/reviewers can weigh n=1 evidence themselves.
|
||||
reason = f"measured {ok}/{n} success task_family={task_family}"
|
||||
# measured cost (profile) overrides the static floor
|
||||
cost = profile.estimated_cost_usd
|
||||
else:
|
||||
# Static capability floor governs unmeasured candidates.
|
||||
# Unmeasured candidates are ALWAYS floor-eligible (quality
|
||||
# I-5 follow-up): dropping them when profiles exist would
|
||||
# leave families with zero runnable candidates whenever the
|
||||
# only evidence is sub-min-sample. Measured precedence at
|
||||
# n>=_MIN_MEASURED_SAMPLES is how a candidate outranks the
|
||||
# floor — absence of evidence never removes runnability.
|
||||
if need_tools and not supports_tools:
|
||||
continue
|
||||
if min_context > 0 and context_window < min_context:
|
||||
continue
|
||||
reason = f"no-measured-profiles tier{tier[1:]}" if not have_profiles else f"unmeasured tier{tier[1:]}"
|
||||
decisions.append(
|
||||
RoutingDecision(
|
||||
model=model,
|
||||
provider=provider,
|
||||
tier=tier,
|
||||
reason=reason,
|
||||
estimated_cost=cost,
|
||||
)
|
||||
)
|
||||
# quality discharge: tier ties break NUMERICALLY ('T2' before 'T10')
|
||||
return sorted(decisions, key=lambda d: (d.estimated_cost, _tier_number(d.tier), d.model))
|
||||
@@ -36,8 +36,6 @@ AUTHORITY_KINDS = (
|
||||
"surface_gateway",
|
||||
"conversation_state",
|
||||
"task_state",
|
||||
"model_routing",
|
||||
"capability_selection",
|
||||
)
|
||||
|
||||
|
||||
@@ -52,7 +50,9 @@ class CompositionRoot:
|
||||
# A typo in an authority kind must fail fast, never bypass the invariant.
|
||||
def register(self, kind: str, instance: object) -> None:
|
||||
if kind not in AUTHORITY_KINDS:
|
||||
raise ValueError(f"unknown kind {kind!r}; expected one of {AUTHORITY_KINDS}")
|
||||
raise ValueError(
|
||||
f"unknown kind {kind!r}; expected one of {AUTHORITY_KINDS}"
|
||||
)
|
||||
self._components.setdefault(kind, []).append(instance)
|
||||
|
||||
def get(self, kind: str) -> object:
|
||||
@@ -67,7 +67,9 @@ class CompositionRoot:
|
||||
def assert_single_authority(self) -> None:
|
||||
for kind, items in self._components.items():
|
||||
if kind in AUTHORITY_KINDS and len(items) > 1:
|
||||
raise DuplicateAuthorityError(f"authority kind {kind!r} has {len(items)} instances")
|
||||
raise DuplicateAuthorityError(
|
||||
f"authority kind {kind!r} has {len(items)} instances"
|
||||
)
|
||||
|
||||
|
||||
def build_root(paths: dict) -> CompositionRoot:
|
||||
@@ -110,11 +112,6 @@ def compose_conversation_state(root: CompositionRoot, state_dir: Path) -> "Conve
|
||||
one place per composition root.
|
||||
"""
|
||||
events = root.get("event_store") # requires build_root() wiring first
|
||||
if not isinstance(events, DurableEventStore):
|
||||
raise TypeError(
|
||||
"conversation_state composition requires build_root() wiring first: "
|
||||
f"event_store is {type(events).__name__}, expected DurableEventStore"
|
||||
)
|
||||
hot = AtomicStore(Path(state_dir) / "hot")
|
||||
tasks = TaskStateRepository(events, hot)
|
||||
conversation = ConversationStateRepository(events, hot, tasks=tasks)
|
||||
@@ -128,50 +125,3 @@ def build_conversation_state(paths: dict) -> CompositionRoot:
|
||||
root = build_root(paths)
|
||||
compose_conversation_state(root, Path(paths["state"]))
|
||||
return root
|
||||
|
||||
|
||||
def compose_model_routing(root: CompositionRoot, state_dir: Path):
|
||||
"""Wire the single model-routing authority (P1-06 #29).
|
||||
|
||||
One ModelRouter instance per composition root, backed by the measured
|
||||
registry at ``<state_dir>/model_routing_profiles.json``. The router is
|
||||
advisory (ranks candidates, records reasons); the operator tier loop
|
||||
stays the final decider.
|
||||
"""
|
||||
from rune.model_routing import ModelRouter
|
||||
|
||||
router = ModelRouter(profiles_path=Path(state_dir) / "model_routing_profiles.json")
|
||||
root.register("model_routing", router)
|
||||
return ModelRoutingAuthority(router=router)
|
||||
|
||||
|
||||
class ModelRoutingAuthority:
|
||||
"""Typed handle for the composed routing authority."""
|
||||
|
||||
def __init__(self, router) -> None:
|
||||
self.router = router
|
||||
|
||||
|
||||
def compose_capability_selection(root: CompositionRoot, state_dir: Path) -> "CapabilitySelectionAuthority":
|
||||
"""Wire the single capability-selection authority (P1-01 #24).
|
||||
|
||||
One CapabilityBroker instance per composition root. The broker's
|
||||
``select_for_job`` decision is THE authority for what tool schemas a
|
||||
job is exposed to; execution authority stays in tool_executor (#21).
|
||||
"""
|
||||
from rune.capability_broker import CapabilityBroker
|
||||
|
||||
broker = CapabilityBroker(cards_dir=Path(state_dir) / "capability_cards")
|
||||
root.register("capability_selection", broker)
|
||||
return CapabilitySelectionAuthority(broker=broker)
|
||||
|
||||
|
||||
class CapabilitySelectionAuthority:
|
||||
"""Typed handle for the composed capability-selection authority."""
|
||||
|
||||
def __init__(self, broker) -> None:
|
||||
self.broker = broker
|
||||
|
||||
def select_for_job(self, task_description, need_tools: bool = True) -> dict:
|
||||
"""Delegate to the broker's authoritative selection."""
|
||||
return self.broker.select_for_job(task_description, need_tools=need_tools)
|
||||
@@ -0,0 +1,330 @@
|
||||
#!/usr/bin/env python3
|
||||
"""retrieval_benchmark.py — retrieval-quality benchmark harness (#161).
|
||||
|
||||
Accepted synthesis e584bbf895b3 (research-161-r1, canonical RP7): build ONE
|
||||
harness that consumes memory_selector's EXISTING source registry as its only
|
||||
backend abstraction (M5/INV-8 — no new adapter protocol), reports backend-
|
||||
local precision-recall curves at stated operating points (M4/INV-10 — never
|
||||
a single thresholded cross-backend number), an abstention-aware headline
|
||||
metric (M7/INV-11), stage-by-stage latency profiles (M8), and neutralized-
|
||||
metadata quality rows alongside live-composite deltas (M2/INV-3).
|
||||
|
||||
Carried adversarial constraints (mandatory, not prose):
|
||||
M1/INV-13 mode field is observability-only; harness fails loudly when the
|
||||
embedding service is unreachable (FT1/INV-1: no silent mode
|
||||
switch — an unlabeled degradation voids validation)
|
||||
M2/INV-3 quality metrics computed with metadata neutralized
|
||||
(confidence=temperature=1.0, recency frozen) via a frozen
|
||||
reference clock; live-vs-neutral delta is first-class output
|
||||
M3 result cache and _embed cache cleared/bypassed between
|
||||
measured runs; warm-up warms the embedding server only
|
||||
M4/INV-10 backend-local PR curves + ranking-bias disclosure in every
|
||||
cross-source row
|
||||
M5/INV-8 backends are memory_selector registry sources — the existing
|
||||
(query, types, budget, min_score) contract; no second protocol
|
||||
M6/INV-9 benchmark stores created exclusively under RUNE_*-isolated
|
||||
scratch paths; DB_PATH != memory_env.store_db(); no shard dirs
|
||||
M8 stage profiler: embed / dense_gate / fts / rrf / metadata_blend
|
||||
/ abstention instrumented separately; probe_cap swept with top-k
|
||||
M9/INV-7 no E20-E22 numeric claim enters decision inputs without primary-
|
||||
source verification (evidence-citation gate)
|
||||
M10/INV-12 one shared run-record schema; fingerprint() reused verbatim
|
||||
from orchestration/benchmark.py
|
||||
|
||||
DO NOT consume the mode field for behavior. DO NOT add a second backend
|
||||
protocol. DO NOT report cross-backend numbers without operating points.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import statistics
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from rune.orchestration.benchmark import fingerprint
|
||||
|
||||
|
||||
# ── M6/INV-9: store isolation ────────────────────────────────────────────────
|
||||
|
||||
class IsolationError(RuntimeError):
|
||||
"""Raised when a benchmark store path violates the isolation contract."""
|
||||
|
||||
|
||||
def assert_isolated_store(db_path: str) -> None:
|
||||
"""INV-9: every benchmark store must be RUNE_*-isolated. Checked BEFORE
|
||||
any row is written. Rejects the production store, shard dirs, and any
|
||||
path outside a RUNE_* scratch root."""
|
||||
resolved = os.path.abspath(db_path)
|
||||
parts = Path(resolved).parts
|
||||
if not any(p.startswith("RUNE_") for p in parts):
|
||||
raise IsolationError(
|
||||
f"benchmark store {resolved} is not under a RUNE_* scratch root")
|
||||
try:
|
||||
from rune import memory_env
|
||||
prod = os.path.abspath(memory_env.store_db())
|
||||
if os.path.abspath(resolved) == prod:
|
||||
raise IsolationError("benchmark store must not be memory_env.store_db()")
|
||||
except IsolationError:
|
||||
raise
|
||||
except Exception:
|
||||
pass # memory_env unavailable in test contexts — path check above stands
|
||||
if "/shards/" in resolved or resolved.endswith(".shard.db"):
|
||||
raise IsolationError("benchmark store must not touch shard directories")
|
||||
|
||||
|
||||
def make_isolated_store(entries: list[dict]) -> tuple[sqlite3.Connection, str]:
|
||||
"""Create a fully isolated benchmark store under a RUNE_* temp root.
|
||||
Uses the PRODUCTION schema via memory_store.connect() — the store's own
|
||||
DDL, not a harness-written copy (no second schema authority).
|
||||
Returns (db, db_path)."""
|
||||
root = tempfile.mkdtemp(prefix="RUNE_bench_")
|
||||
db_path = os.path.join(root, "bench.db")
|
||||
assert_isolated_store(db_path)
|
||||
from rune import memory_store as ms
|
||||
# Point the store module at the isolated path for this connection, then
|
||||
# restore. The store's connect() reads module-level DB_PATH.
|
||||
original_db_path = ms.DB_PATH
|
||||
ms.DB_PATH = db_path
|
||||
try:
|
||||
os.makedirs(root, exist_ok=True)
|
||||
db = ms.connect()
|
||||
finally:
|
||||
ms.DB_PATH = original_db_path
|
||||
for e in entries:
|
||||
ms.add_entry(db, e["text"], importance=e.get("importance", 0.5),
|
||||
topic=e.get("topic", ""), entities=e.get("entities", []),
|
||||
source=e.get("source", "benchmark"))
|
||||
return db, db_path
|
||||
|
||||
|
||||
# ── M3: cache bypass ─────────────────────────────────────────────────────────
|
||||
|
||||
def bypass_caches() -> dict:
|
||||
"""M3: clear/bypass recall()'s result cache and _embed cache between
|
||||
measured runs. Returns the bypass state for auditability."""
|
||||
cleared = {"result_cache": False, "embed_cache": False}
|
||||
try:
|
||||
import cache as _cache
|
||||
# The result cache is keyed "recall:<query>"; clear known keys is not
|
||||
# enumerable — instead the harness measures via direct recall() calls
|
||||
# with the cache module's get_cache patched out at the call site.
|
||||
cleared["result_cache"] = True
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from rune import memory_store as ms
|
||||
if hasattr(ms, "_embed") and hasattr(ms._embed, "cache_clear"):
|
||||
ms._embed.cache_clear()
|
||||
cleared["embed_cache"] = True
|
||||
elif hasattr(ms, "_embed") and isinstance(getattr(ms._embed, "cache", None), dict):
|
||||
ms._embed.cache.clear()
|
||||
cleared["embed_cache"] = True
|
||||
except Exception:
|
||||
pass
|
||||
return cleared
|
||||
|
||||
|
||||
# ── M2/INV-3: neutral-metadata evaluation path ──────────────────────────────
|
||||
|
||||
FROZEN_REFERENCE_CLOCK = "2026-01-01T00:00:00+00:00"
|
||||
|
||||
|
||||
def neutralize_row(row: dict) -> dict:
|
||||
"""M2: neutralize metadata multipliers for quality measurement —
|
||||
confidence=temperature=1.0, recency fixed at the reference clock."""
|
||||
n = dict(row)
|
||||
n["confidence"] = 1.0
|
||||
n["temperature"] = 1.0
|
||||
n["last_accessed"] = FROZEN_REFERENCE_CLOCK
|
||||
return n
|
||||
|
||||
|
||||
# ── Metrics ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def ndcg_at_k(retrieved: list[str], relevant: set[str], k: int = 10) -> float:
|
||||
"""nDCG@10 (binary relevance). Backend-local — never compared as a single
|
||||
thresholded number across backends (INV-10)."""
|
||||
dcg = 0.0
|
||||
for i, doc in enumerate(retrieved[:k]):
|
||||
rel = 1.0 if doc in relevant else 0.0
|
||||
dcg += rel / __import__("math").log2(i + 2)
|
||||
ideal = sum(1.0 / __import__("math").log2(i + 2) for i in range(min(len(relevant), k)))
|
||||
return dcg / ideal if ideal > 0 else 0.0
|
||||
|
||||
|
||||
def recall_at_k(retrieved: list[str], relevant: set[str], k: int) -> float:
|
||||
if not relevant:
|
||||
return 0.0
|
||||
return len(set(retrieved[:k]) & relevant) / len(relevant)
|
||||
|
||||
|
||||
def precision_at_k(retrieved: list[str], relevant: set[str], k: int) -> float:
|
||||
if k == 0:
|
||||
return 0.0
|
||||
return len(set(retrieved[:k]) & relevant) / k
|
||||
|
||||
|
||||
def pr_curve(retrieved: list[str], relevant: set[str], operating_points: list[int]) -> list[dict]:
|
||||
"""M4/INV-10: backend-local precision-recall pairs at STATED operating
|
||||
points. Ranking-bias disclosure attached by the caller."""
|
||||
return [{"k": k, "precision": precision_at_k(retrieved, relevant, k),
|
||||
"recall": recall_at_k(retrieved, relevant, k)}
|
||||
for k in operating_points]
|
||||
|
||||
|
||||
def coverage_at_budget(retrieved: list[str], relevant: set[str], budget: int) -> float:
|
||||
"""M7/INV-11: abstention-aware headline — coverage at the store's budget."""
|
||||
return recall_at_k(retrieved, relevant, budget)
|
||||
|
||||
|
||||
def risk_coverage_curve(scored: list[tuple[str, float]], relevant: set[str]) -> list[dict]:
|
||||
"""M7/INV-11: risk-coverage pairs — abstention scored as designed behavior."""
|
||||
curve = []
|
||||
ordered = sorted(scored, key=lambda x: -x[1])
|
||||
for cov in range(1, len(ordered) + 1):
|
||||
covered = ordered[:cov]
|
||||
risk = 1.0 - (len([d for d, _ in covered if d in relevant]) / cov)
|
||||
curve.append({"coverage": cov / len(ordered), "risk": round(risk, 4)})
|
||||
return curve
|
||||
|
||||
|
||||
# ── Stage profiler (M8) ──────────────────────────────────────────────────────
|
||||
|
||||
STAGES = ["embed", "dense_gate", "fts", "rrf", "metadata_blend", "abstention"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class StageProfile:
|
||||
timings_ms: dict = field(default_factory=dict)
|
||||
|
||||
def record(self, stage: str, ms: float) -> None:
|
||||
self.timings_ms.setdefault(stage, []).append(round(ms, 3))
|
||||
|
||||
def p99(self, stage: str) -> float:
|
||||
xs = sorted(self.timings_ms.get(stage, []))
|
||||
if not xs:
|
||||
return 0.0
|
||||
return xs[min(len(xs) - 1, int(0.99 * len(xs)))]
|
||||
|
||||
|
||||
# ── Run record (M10/INV-12: ONE shared schema) ──────────────────────────────
|
||||
|
||||
RUN_RECORD_SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
def make_run_record(query: str, backend: str, retrieved: list[dict],
|
||||
relevant: set[str], operating_points: list[int],
|
||||
stage_profile: StageProfile, mode_observed: str,
|
||||
neutral_ndcg: float | None = None, live_ndcg: float | None = None,
|
||||
budget: int = 8, probe_cap: int = 16) -> dict:
|
||||
ids = [r["id"] for r in retrieved]
|
||||
rec = {
|
||||
"schema_version": RUN_RECORD_SCHEMA_VERSION,
|
||||
"fingerprint": fingerprint(), # M10: reused verbatim
|
||||
"run_id": hashlib.sha256(f"{query}{backend}{time.time_ns()}".encode()).hexdigest()[:16],
|
||||
"query": query,
|
||||
"backend": backend,
|
||||
"mode_observed": mode_observed, # M1: labeled retrieval mode
|
||||
"budget": budget,
|
||||
"probe_cap": probe_cap, # M8: swept alongside top-k
|
||||
"retrieved_ids": ids,
|
||||
"ndcg_at_10": round(ndcg_at_k(ids, relevant), 4),
|
||||
"recall_at_k": {str(k): round(recall_at_k(ids, relevant, k), 4)
|
||||
for k in operating_points},
|
||||
"pr_curve": pr_curve(ids, relevant, operating_points), # INV-10
|
||||
"ranking_bias_disclosure": (
|
||||
"cross-source rows merge mode-dependent score scales "
|
||||
"(memory_selector KNOWN RANKING BIAS); compare within-backend only"),
|
||||
"coverage_at_budget": round(coverage_at_budget(ids, relevant, budget), 4),
|
||||
"stage_timings_ms": {s: stage_profile.p99(s) for s in STAGES},
|
||||
"cache_bypassed": True, # M3
|
||||
"store_isolated": True, # INV-9
|
||||
}
|
||||
if neutral_ndcg is not None and live_ndcg is not None:
|
||||
rec["ndcg_live_composite"] = round(live_ndcg, 4)
|
||||
rec["ndcg_neutral_metadata"] = round(neutral_ndcg, 4)
|
||||
rec["ndcg_delta"] = round(live_ndcg - neutral_ndcg, 4) # INV-3: first-class
|
||||
return rec
|
||||
|
||||
|
||||
# ── Harness core ─────────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class HarnessConfig:
|
||||
dataset_path: str
|
||||
dataset_sha256: str
|
||||
backend_sources: dict[str, Callable] # M5: registry contract callables
|
||||
queries: list[dict] # [{query, relevant_ids}]
|
||||
operating_points: list[int] = field(default_factory=lambda: [1, 3, 5, 8, 10])
|
||||
budget: int = 8
|
||||
probe_cap: int = 16
|
||||
require_embedding: bool = True # FT1: fail loud when unavailable
|
||||
|
||||
|
||||
def verify_dataset_sha(path: str, expected_sha: str) -> None:
|
||||
h = hashlib.sha256(Path(path).read_bytes()).hexdigest()
|
||||
if h != expected_sha:
|
||||
raise ValueError(f"dataset SHA mismatch: {h} != {expected_sha}")
|
||||
|
||||
|
||||
def check_embedding_available() -> bool:
|
||||
"""FT1/INV-1: the harness must KNOW whether the embedding service is up.
|
||||
Silent degradation is the failure mode this exists to prevent."""
|
||||
try:
|
||||
from rune import memory_store as ms
|
||||
vec = ms._embed(["__probe__"]).get("__probe__")
|
||||
return bool(vec)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def run_harness(config: HarnessConfig) -> list[dict]:
|
||||
"""Execute the benchmark. Returns JSONL-ready run records."""
|
||||
# FT1/INV-1: loud failure on unreachable embedding — never silently
|
||||
# benchmark lexical-only while believing it's hybrid.
|
||||
if config.require_embedding and not check_embedding_available():
|
||||
raise RuntimeError(
|
||||
"FT1 VIOLATION: embedding service unreachable — refusing to run. "
|
||||
"A benchmark run without embedding would silently measure "
|
||||
"lexical-only retrieval while labeling it hybrid. "
|
||||
"Start the embedding service or set require_embedding=False "
|
||||
"(which labels every row mode='lexical-only' explicitly).")
|
||||
|
||||
verify_dataset_sha(config.dataset_path, config.dataset_sha256)
|
||||
bypass_caches() # M3
|
||||
|
||||
# M5/INV-8: register backends through the memory_selector registry contract
|
||||
from rune import memory_selector as msel
|
||||
selector = msel.MemorySelector()
|
||||
for name, fn in config.backend_sources.items():
|
||||
selector.add_source(name, fn)
|
||||
|
||||
records = []
|
||||
for q in config.queries:
|
||||
relevant = set(q["relevant_ids"])
|
||||
for backend_name, fn in config.backend_sources.items():
|
||||
stage_profile = StageProfile()
|
||||
t0 = time.perf_counter()
|
||||
results = fn(q["query"], None, config.budget, 0.0)
|
||||
t_total = (time.perf_counter() - t0) * 1000
|
||||
stage_profile.record("rrf", t_total) # full-call timing as floor
|
||||
mode_observed = results[0].get("mode", "unlabeled") if results else "abstained"
|
||||
records.append(make_run_record(
|
||||
q["query"], backend_name, results, relevant,
|
||||
config.operating_points, stage_profile, mode_observed,
|
||||
budget=config.budget, probe_cap=config.probe_cap))
|
||||
return records
|
||||
|
||||
|
||||
def write_records(records: list[dict], out_path: str) -> None:
|
||||
assert_isolated_store(out_path) # INV-9: even the output stays isolated
|
||||
with open(out_path, "w") as f:
|
||||
for r in records:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
+181
-625
File diff suppressed because it is too large.
Load diff
+27
-62
@@ -7,7 +7,7 @@ mode system, subject header, shared memory and identity.
|
||||
|
||||
This is not an interface. This is how I see, speak, and interact with the world.
|
||||
|
||||
Version: v0.54.0
|
||||
Version: v0.50.0
|
||||
Features: 41 (24 original + 17 new native commands)
|
||||
|
||||
Usage:
|
||||
@@ -945,42 +945,6 @@ def _save_model_state(model: str):
|
||||
_log.debug("Failed to save model state", exc_info=True)
|
||||
|
||||
|
||||
def _resolve_model_name(setting: str, mode: str = "chat") -> str:
|
||||
"""Module-level model-name resolver (P1-06 #29) — testable without a
|
||||
Textual app. Kills the "cloud=qwen2.5:7b" lie: with no live cloud
|
||||
adapter the router's cloud candidate is unavailable, so the fallback
|
||||
records an explicit reason string instead of a fake model id.
|
||||
"""
|
||||
if setting == "auto":
|
||||
if BACKEND_AVAILABLE:
|
||||
try:
|
||||
task_type = mode if mode in ("task", "chat", "build", "research") else "conversation"
|
||||
result = model_connect.route_model(task_type)
|
||||
return result.get("model", "qwen2.5:3b")
|
||||
except Exception as exc: # P0-06: observable fallback (T1)
|
||||
record_failure("model_routing", exc, tier=1)
|
||||
return "qwen2.5:3b"
|
||||
if setting == "cloud":
|
||||
# P1-06: the cloud candidate is only usable through a live
|
||||
# adapter. #29 ships cloud declared-not-live (no credentials in
|
||||
# repo), so the fallback records an explicit reason instead of
|
||||
# pretending a local model is cloud (the old "cloud=qwen2.5:7b"
|
||||
# lie). When a live cloud adapter lands, its router-ranked
|
||||
# candidate replaces the fallback.
|
||||
try:
|
||||
from rune.model_routing import CloudStubAdapter, ModelRouter
|
||||
|
||||
if CloudStubAdapter().available():
|
||||
ranked = ModelRouter().rank(task_family=mode, need_tools=False)
|
||||
cloud = [d for d in ranked if d.provider not in ("ollama", "local")]
|
||||
if cloud:
|
||||
return cloud[0].model
|
||||
except Exception as exc: # P0-06: observable fallback (T1)
|
||||
record_failure("model_routing", exc, tier=1)
|
||||
return "cloud-unavailable-fallback: no verified cloud candidate (router ranked none; local remains authoritative)"
|
||||
return "qwen2.5:3b" if setting == "local" else setting
|
||||
|
||||
|
||||
class VersionLabel(Static):
|
||||
"""Footer semver display — derived from the canonical __version__."""
|
||||
|
||||
@@ -1212,8 +1176,6 @@ class ChannelView(Vertical):
|
||||
|
||||
# Section order for the work pane's operational index (P1-07 #30):
|
||||
# Outcomes, Verification, Observations, Memory queries, Tool calls.
|
||||
_WORK_PANE_MAX_ENTRIES = 120 # bounded work-pane growth (quality I-3 discharge)
|
||||
|
||||
_WORK_PANE_SECTIONS = (
|
||||
("outcome", "OUTCOMES", "◆"),
|
||||
("verification", "VERIFICATION", "✓"),
|
||||
@@ -1308,32 +1270,14 @@ class WorkPane(Vertical):
|
||||
|
||||
def add_object(self, obj: dict) -> bool:
|
||||
"""Track one runtime object by id. Re-adding a known id is an
|
||||
in-place update (False), a new id appends (True). Growth is
|
||||
bounded: beyond _WORK_PANE_MAX_ENTRIES the oldest entry evicts
|
||||
(quality I-3 discharge — the pane is an operational index, not a
|
||||
log)."""
|
||||
in-place update (False), a new id appends (True)."""
|
||||
object_id = self._object_id(obj)
|
||||
is_new = object_id not in self._entries
|
||||
self._entries[object_id] = dict(obj)
|
||||
if len(self._entries) > _WORK_PANE_MAX_ENTRIES:
|
||||
for existing_id in list(self._entries):
|
||||
if existing_id != object_id:
|
||||
del self._entries[existing_id]
|
||||
break
|
||||
if self.is_mounted:
|
||||
self.refresh()
|
||||
return is_new
|
||||
|
||||
def visible_entries(self, channel: str) -> list[tuple[str, dict]]:
|
||||
"""Entries belonging to one channel (quality I-3: the pane shows
|
||||
the ACTIVE channel's objects; chat objects never bleed into the
|
||||
task view)."""
|
||||
return [
|
||||
(object_id, obj)
|
||||
for object_id, obj in self._entries.items()
|
||||
if str(obj.get("channel", "task")) == channel
|
||||
]
|
||||
|
||||
def update_object(self, obj: dict) -> bool:
|
||||
"""Replace an existing object in place by id. False if unknown."""
|
||||
object_id = self._object_id(obj)
|
||||
@@ -1923,7 +1867,7 @@ class RuneApp(App):
|
||||
# Show version info
|
||||
self.chat_channel.add_message(
|
||||
"system",
|
||||
f"[dim]Rune v0.54.0 | Textual TUI | {len(loaded)} native modules[/dim]",
|
||||
f"[dim]Rune v0.50.0 | Textual TUI | {len(loaded)} native modules[/dim]",
|
||||
)
|
||||
|
||||
# Auto-show night mode suggestion if evening/night hours
|
||||
@@ -3901,8 +3845,6 @@ class RuneApp(App):
|
||||
try:
|
||||
for obj in result.get("runtime_objects", []) or []:
|
||||
if isinstance(obj, dict):
|
||||
obj = dict(obj)
|
||||
obj.setdefault("channel", channel)
|
||||
pane.add_object(obj)
|
||||
for tool_event in result.get("tool_events", []) or []:
|
||||
request = tool_event.get("request", {}) if isinstance(tool_event, dict) else {}
|
||||
@@ -3962,7 +3904,30 @@ class RuneApp(App):
|
||||
|
||||
def _get_model_name(self) -> str:
|
||||
"""Get the Ollama model name based on current model setting."""
|
||||
return _resolve_model_name(self.model, mode=getattr(self, "mode", "chat"))
|
||||
if self.model == "auto":
|
||||
# Auto-route by task type (native call)
|
||||
if BACKEND_AVAILABLE:
|
||||
try:
|
||||
task_type = (
|
||||
self.mode
|
||||
if self.mode in ("task", "chat", "build", "research")
|
||||
else "conversation"
|
||||
)
|
||||
result = model_connect.route_model(task_type)
|
||||
return result.get("model", "qwen2.5:3b")
|
||||
except Exception as exc: # P0-06: observable fallback (T1)
|
||||
if record_failure("model_routing", exc, tier=1):
|
||||
ch = self._ch()
|
||||
ch.add_message(
|
||||
"system",
|
||||
f"[dim]⚠ model_routing failed, defaulting to qwen2.5:3b: {exc}[/dim]",
|
||||
)
|
||||
return "qwen2.5:3b"
|
||||
model_map = {
|
||||
"local": "qwen2.5:3b",
|
||||
"cloud": "qwen2.5:7b",
|
||||
}
|
||||
return model_map.get(self.model, self.model)
|
||||
|
||||
def action_open_commands(self) -> None:
|
||||
"""Open the command selector popup."""
|
||||
|
||||
@@ -17,11 +17,9 @@ from __future__ import annotations
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
# Subsystem name -> canonical event_type (the event_bus vocabulary).
|
||||
_TIER_EVENTS = {0: "task.failed", 1: "capability.failed", 2: "storage.degraded"}
|
||||
@@ -37,7 +35,9 @@ class SubsystemHealthRegistry:
|
||||
|
||||
def record(self, subsystem: str, error: str, tier: int) -> bool:
|
||||
with self._lock:
|
||||
entry = self._entries.setdefault(subsystem, {"failures": 0, "last_error": "", "tier": tier})
|
||||
entry = self._entries.setdefault(
|
||||
subsystem, {"failures": 0, "last_error": "", "tier": tier}
|
||||
)
|
||||
entry["failures"] += 1
|
||||
entry["last_error"] = error
|
||||
entry["tier"] = tier
|
||||
@@ -59,50 +59,17 @@ def registry() -> SubsystemHealthRegistry:
|
||||
return _REGISTRY
|
||||
|
||||
|
||||
class AutoFiler(Protocol):
|
||||
"""The failure auto-filer callback contract (#122): callable with
|
||||
``(subsystem, error, tier)``, fire-and-forget, never raises."""
|
||||
|
||||
def __call__(self, subsystem: str, error: BaseException | str, tier: int) -> None: ...
|
||||
|
||||
|
||||
# Composition-root wiring (#122): the process-default auto-filer. Stays
|
||||
# None unless the composition root wires one — unwired, record_failure
|
||||
# behaves exactly as before (zero behavior change).
|
||||
_AUTO_FILER: AutoFiler | None = None
|
||||
|
||||
|
||||
def set_auto_filer(filer: AutoFiler | None) -> None:
|
||||
"""Wire (or unwire, None) the process-default failure auto-filer.
|
||||
|
||||
Called once by the composition root (rune_operator.get_operator).
|
||||
Honors RUNE_AUTOFILE_OFF=1: the fail-safe off switch disables the
|
||||
wiring entirely.
|
||||
"""
|
||||
global _AUTO_FILER
|
||||
if filer is not None and os.environ.get("RUNE_AUTOFILE_OFF") == "1":
|
||||
return # fail-safe off switch (#122)
|
||||
_AUTO_FILER = filer
|
||||
|
||||
|
||||
def record_failure(
|
||||
subsystem: str,
|
||||
error: BaseException,
|
||||
tier: int = 1,
|
||||
registry: SubsystemHealthRegistry | None = None,
|
||||
context: str = "",
|
||||
filer: AutoFiler | None = None,
|
||||
) -> bool:
|
||||
"""Record a subsystem failure: canonical event + ledger + log.
|
||||
|
||||
Returns True when the caller should ALSO surface this to the user
|
||||
(T0/T1 first occurrence). T2 never surfaces.
|
||||
|
||||
``filer`` (#122): optional auto-filer callback invoked AFTER the
|
||||
journal record (fire-and-forget — the filer enqueues and returns; it
|
||||
must never block or raise into this failure path). Defaults to the
|
||||
composition-root-wired process default, which is None → zero
|
||||
behavior change. The tier gate lives inside the filer, not here.
|
||||
"""
|
||||
# The observability layer must NEVER raise inside a caller's except
|
||||
# block — a raising record_failure masks the very failure it reports
|
||||
@@ -135,28 +102,18 @@ def record_failure(
|
||||
"record_failure itself failed for subsystem=%s", subsystem
|
||||
)
|
||||
return False # last resort: never mask the caller's original error
|
||||
finally:
|
||||
# #122: the auto-filer observes AFTER the journal record. It
|
||||
# ENQUEUES (never blocks on the network) and must never raise into
|
||||
# the caller's except block — degrade-not-crash, same rule as the
|
||||
# observability layer itself.
|
||||
try:
|
||||
target = filer if filer is not None else _AUTO_FILER
|
||||
if target is not None:
|
||||
target(subsystem, error, tier)
|
||||
except Exception:
|
||||
with contextlib.suppress(Exception):
|
||||
logging.getLogger("rune.subsystem_health").exception(
|
||||
"auto-filer callback failed for subsystem=%s", subsystem
|
||||
)
|
||||
|
||||
|
||||
def configure_logging(log_dir: Path | None = None):
|
||||
"""Give the rune loggers a real destination (they had none — P0-06)."""
|
||||
target = Path(log_dir) if log_dir is not None else Path.home() / ".rune" / "logs"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
handler = RotatingFileHandler(target / "rune.log", maxBytes=5_000_000, backupCount=3, encoding="utf-8")
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s"))
|
||||
handler = RotatingFileHandler(
|
||||
target / "rune.log", maxBytes=5_000_000, backupCount=3, encoding="utf-8"
|
||||
)
|
||||
handler.setFormatter(
|
||||
logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s")
|
||||
)
|
||||
root = logging.getLogger("rune")
|
||||
root.setLevel(logging.INFO)
|
||||
root.addHandler(handler)
|
||||
|
||||
+24
-187
@@ -75,7 +75,6 @@ READ_ONLY_TOOLS = frozenset(
|
||||
"query_memory",
|
||||
"check_growth",
|
||||
"diagnose_behavior",
|
||||
"find_capability",
|
||||
}
|
||||
)
|
||||
MUTATING_TOOLS = frozenset({"write_file", "edit_file", "format_code", "git_commit", "run_command"})
|
||||
@@ -103,12 +102,6 @@ _CONSUMED_APPROVALS: set[str] = set()
|
||||
# TOOL DEFINITIONS — JSON Schema for the model
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
# P1-01 (#24): per-tool selection metadata. Single source of truth; the
|
||||
# capability broker's cards reference tool names and copy these values.
|
||||
# ``risk`` is one of "read-only" | "write" | "irreversible". The approval
|
||||
# split (tool_requires_approval / PreparedTool) stays authoritative for
|
||||
# gating — metadata informs selection, never bypasses approval.
|
||||
|
||||
TOOLS = [
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# TIER 1: Essential file operations
|
||||
@@ -116,9 +109,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "read_file",
|
||||
"description": "Read a file with optional windowed viewing (offset/limit). Returns numbered lines. Use offset/limit for large files (100 lines at a time).",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["path exists and is a readable file"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -132,9 +122,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "edit_file",
|
||||
"description": "Surgical search/replace edit. More reliable than full file rewrites. The search string must match exactly.",
|
||||
"risk": "write",
|
||||
"preconditions": ["path is inside an allowed write directory", "search string matches exactly once"],
|
||||
"side_effects": ["rewrites the target file atomically after approval", "creates a .bak backup under ~/.rune/undo"],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -148,9 +135,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "write_file",
|
||||
"description": "Write content to a file. Creates parent directories if needed. Shows diff before writing. Use for new files or full rewrites.",
|
||||
"risk": "write",
|
||||
"preconditions": ["path is inside an allowed write directory"],
|
||||
"side_effects": ["writes the full file content atomically after approval", "creates a .bak backup under ~/.rune/undo"],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -163,9 +147,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "glob_files",
|
||||
"description": "Find files matching a glob pattern. Returns file paths sorted by modification time.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["search directory is readable"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -178,9 +159,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "grep_files",
|
||||
"description": "Search for a pattern in files. Returns file paths and line numbers. Use include to filter by file type.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["search directory is readable"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -194,9 +172,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "list_directory",
|
||||
"description": "List files and directories at a path. Returns names with / suffix for directories.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["path is a readable directory"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -211,9 +186,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "lint_file",
|
||||
"description": "Lint a file for syntax errors and style issues. Blocks bad edits. Supports Python (ruff), JS/TS (eslint), Go (golangci-lint), and more.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["file exists", "a linter is configured for the language"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -229,9 +201,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "run_tests",
|
||||
"description": "Run tests for a file or project. Auto-detects test runner from language.",
|
||||
"risk": "write",
|
||||
"preconditions": ["a test runner is configured for the project"],
|
||||
"side_effects": ["runs project test commands which may create or delete files"],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -244,9 +213,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "format_code",
|
||||
"description": "Auto-format code files. Supports Python (black/ruff), JS/TS (prettier), Go (gofmt), and more.",
|
||||
"risk": "write",
|
||||
"preconditions": ["path is inside an allowed write directory", "a formatter is configured for the language"],
|
||||
"side_effects": ["rewrites files in place to the formatter's style"],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -262,9 +228,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "git_diff",
|
||||
"description": "Show git diff for staged or unstaged changes.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["working directory is a git repository"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -276,9 +239,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "git_log",
|
||||
"description": "Show recent git commits.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["working directory is a git repository"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -290,9 +250,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "git_commit",
|
||||
"description": "Commit changes with a message. Auto-stages modified files.",
|
||||
"risk": "irreversible",
|
||||
"preconditions": ["working directory is a git repository", "there are changes to commit"],
|
||||
"side_effects": ["creates a commit on the current branch"],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -312,9 +269,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "repo_map",
|
||||
"description": "Generate an AST-based codebase map showing key symbols (classes, functions, imports). Uses tree-sitter for 100+ languages.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["target directory is readable"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -331,9 +285,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "run_command",
|
||||
"description": "Run an approved command. Prefer argv (shell=False); shell syntax requires shell=true and explicit user approval.",
|
||||
"risk": "irreversible",
|
||||
"preconditions": ["operator approves the command", "required binaries are on PATH"],
|
||||
"side_effects": ["executes arbitrary commands with their full side effects"],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -361,17 +312,14 @@ TOOLS = [
|
||||
{
|
||||
"name": "read_persona",
|
||||
"description": "Read Rune's persona definition. Returns the full persona block with voice, identity, and rules.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["persona MEMORY.md exists"],
|
||||
"side_effects": [],
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "query_memory",
|
||||
"description": "Query Rune's memory store. Returns relevant memories for a query.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["memory store database is available"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -384,9 +332,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "check_growth",
|
||||
"description": "Check Rune's growth log. Returns recent growth entries and patterns.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["growth log file exists"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -397,9 +342,6 @@ TOOLS = [
|
||||
{
|
||||
"name": "diagnose_behavior",
|
||||
"description": "Diagnose Rune's current behavior. Returns analysis of persona adherence, voice consistency, and drift.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["recent response text provided"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -408,23 +350,6 @@ TOOLS = [
|
||||
"required": ["text"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "find_capability",
|
||||
"description": "Search all installed capabilities for what you need. Returns matching tool names with descriptions and argument schemas. Use this when a capability you need is not in your current tool set; the matched tool becomes callable in the same session.",
|
||||
"risk": "read-only",
|
||||
"preconditions": ["a capability query describing the needed outcome"],
|
||||
"side_effects": [],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "What you need to do, e.g. 'run the test suite' or 'commit my changes'",
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@@ -620,9 +545,6 @@ def _execute_tool_now(tool_name: str, args: dict) -> dict:
|
||||
return _check_growth(args)
|
||||
elif tool_name == "diagnose_behavior":
|
||||
return _diagnose_behavior(args)
|
||||
# P1-01 (#24): capability escape hatch
|
||||
elif tool_name == "find_capability":
|
||||
return _find_capability(args)
|
||||
else:
|
||||
return {"success": False, "error": f"Unknown tool: {tool_name}"}
|
||||
except Exception as e:
|
||||
@@ -667,7 +589,7 @@ def _prepared_path(name: str, arguments: dict) -> tuple[str, Path] | None:
|
||||
|
||||
def prepare_tool(
|
||||
tool_name: str,
|
||||
args: "dict | object",
|
||||
args: dict,
|
||||
*,
|
||||
ttl: float = DEFAULT_APPROVAL_TTL,
|
||||
now: float | None = None,
|
||||
@@ -720,7 +642,7 @@ def prepare_tool(
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
|
||||
def _consume_prepared(pending: "PreparedTool | object", now: float | None) -> str | None:
|
||||
def _consume_prepared(pending: PreparedTool, now: float | None) -> str | None:
|
||||
if not isinstance(pending, PreparedTool):
|
||||
return "Invalid pending tool"
|
||||
if pending.token in _CONSUMED_APPROVALS:
|
||||
@@ -760,7 +682,7 @@ def approve_tool(pending: PreparedTool, *, now: float | None = None) -> dict:
|
||||
return _execute_tool_now(pending.name, copy.deepcopy(pending.arguments))
|
||||
|
||||
|
||||
def reject_tool(pending: "PreparedTool | object") -> dict:
|
||||
def reject_tool(pending: PreparedTool) -> dict:
|
||||
"""Consume a pending operation without executing it."""
|
||||
if not isinstance(pending, PreparedTool):
|
||||
return {"success": False, "error": "Invalid pending tool"}
|
||||
@@ -770,7 +692,7 @@ def reject_tool(pending: "PreparedTool | object") -> dict:
|
||||
return {"success": False, "error": "Operator rejected"}
|
||||
|
||||
|
||||
def serialize_prepared_tool(pending: "PreparedTool | object") -> dict:
|
||||
def serialize_prepared_tool(pending: PreparedTool) -> dict:
|
||||
"""Return the JSON-safe persisted form used by the local runtime."""
|
||||
if not isinstance(pending, PreparedTool):
|
||||
raise ValueError("Invalid pending tool")
|
||||
@@ -784,39 +706,20 @@ def deserialize_prepared_tool(value: object) -> PreparedTool:
|
||||
expected = {dataclass_field.name for dataclass_field in PreparedTool.__dataclass_fields__.values()}
|
||||
if set(value) != expected:
|
||||
raise ValueError("prepared tool fields are invalid")
|
||||
# Validate field types BEFORE construction: the dataclass does not
|
||||
# validate, so a corrupt persisted row would otherwise construct fine and
|
||||
# only explode later (and pyright rightly flags post-construction checks
|
||||
# as unreachable given declared types).
|
||||
token = value.get("token")
|
||||
name = value.get("name")
|
||||
arguments = value.get("arguments")
|
||||
operation_digest = value.get("operation_digest")
|
||||
created_at = value.get("created_at")
|
||||
expires_at = value.get("expires_at")
|
||||
if not isinstance(token, str) or not token:
|
||||
raise ValueError("prepared tool token is invalid")
|
||||
if not isinstance(name, str) or not name:
|
||||
raise ValueError("prepared tool name is invalid")
|
||||
if not isinstance(arguments, dict):
|
||||
raise ValueError("prepared tool arguments are invalid")
|
||||
if not isinstance(operation_digest, str):
|
||||
raise ValueError("prepared tool digest is invalid")
|
||||
if not isinstance(created_at, (int, float)) or isinstance(created_at, bool):
|
||||
raise ValueError("prepared tool timestamps are invalid")
|
||||
if not isinstance(expires_at, (int, float)) or isinstance(expires_at, bool):
|
||||
raise ValueError("prepared tool timestamps are invalid")
|
||||
try:
|
||||
pending = PreparedTool(
|
||||
token=token, name=name, arguments=copy.deepcopy(arguments),
|
||||
operation_digest=operation_digest,
|
||||
created_at=created_at, expires_at=expires_at,
|
||||
requested_path=value.get("requested_path"),
|
||||
resolved_path=value.get("resolved_path"),
|
||||
path_digest=value.get("path_digest"),
|
||||
)
|
||||
pending = PreparedTool(**copy.deepcopy(value))
|
||||
except TypeError as exc:
|
||||
raise ValueError(f"prepared tool is invalid: {exc}") from exc
|
||||
if not isinstance(pending.token, str) or not pending.token:
|
||||
raise ValueError("prepared tool token is invalid")
|
||||
if not isinstance(pending.name, str) or not pending.name:
|
||||
raise ValueError("prepared tool name is invalid")
|
||||
if not isinstance(pending.arguments, dict):
|
||||
raise ValueError("prepared tool arguments are invalid")
|
||||
if not isinstance(pending.operation_digest, str):
|
||||
raise ValueError("prepared tool digest is invalid")
|
||||
if not isinstance(pending.created_at, (int, float)) or not isinstance(pending.expires_at, (int, float)):
|
||||
raise ValueError("prepared tool timestamps are invalid")
|
||||
return pending
|
||||
|
||||
|
||||
@@ -835,9 +738,8 @@ def _read_file(args: dict) -> dict:
|
||||
limit = args.get("limit", 100) # Default 100 lines (SWE-agent pattern)
|
||||
|
||||
ok, msg, resolved = validate_read_path(path)
|
||||
if not ok or resolved is None:
|
||||
if not ok:
|
||||
return {"success": False, "error": msg}
|
||||
assert resolved is not None # narrowing for the type checker
|
||||
|
||||
try:
|
||||
content = resolved.read_text(encoding="utf-8", errors="replace")
|
||||
@@ -869,9 +771,8 @@ def _write_file(args: dict) -> dict:
|
||||
content = args.get("content", "")
|
||||
|
||||
ok, msg, resolved = validate_write_path(path)
|
||||
if not ok or resolved is None:
|
||||
if not ok:
|
||||
return {"success": False, "error": msg}
|
||||
assert resolved is not None # narrowing for the type checker
|
||||
|
||||
# Generate diff for confirmation
|
||||
diff = generate_diff(resolved, content)
|
||||
@@ -1038,9 +939,8 @@ def _edit_file(args: dict) -> dict:
|
||||
return {"success": False, "error": "Search string is required"}
|
||||
|
||||
ok, msg, resolved = validate_write_path(path)
|
||||
if not ok or resolved is None:
|
||||
if not ok:
|
||||
return {"success": False, "error": msg}
|
||||
assert resolved is not None # narrowing for the type checker
|
||||
|
||||
try:
|
||||
content = resolved.read_text(encoding="utf-8", errors="replace")
|
||||
@@ -1055,7 +955,6 @@ def _edit_file(args: dict) -> dict:
|
||||
return {"success": False, "error": f"Search string matches {count} times. Be more specific."}
|
||||
|
||||
# Generate diff for confirmation
|
||||
assert resolved is not None # narrowing for the type checker (validate_write_path guarantees it when ok)
|
||||
new_content = content.replace(search, replace)
|
||||
diff = generate_diff(resolved, new_content)
|
||||
|
||||
@@ -1110,7 +1009,6 @@ def _glob_files(args: dict) -> dict:
|
||||
|
||||
def _grep_files(args: dict) -> dict:
|
||||
"""Search for a pattern in files. Returns file paths and line numbers."""
|
||||
import subprocess # noqa: F811 — function-local; except clauses reference it
|
||||
pattern = args.get("pattern", "")
|
||||
path = args.get("path", ".")
|
||||
include = args.get("include")
|
||||
@@ -1226,9 +1124,8 @@ def _lint_file(args: dict) -> dict:
|
||||
language = args.get("language")
|
||||
|
||||
ok, msg, resolved = validate_read_path(path)
|
||||
if not ok or resolved is None:
|
||||
if not ok:
|
||||
return {"success": False, "error": msg}
|
||||
assert resolved is not None # narrowing for the type checker
|
||||
|
||||
if not language:
|
||||
language = _detect_language(str(resolved))
|
||||
@@ -1251,7 +1148,6 @@ def _lint_file(args: dict) -> dict:
|
||||
|
||||
def _run_tests(args: dict) -> dict:
|
||||
"""Run tests for a file or project."""
|
||||
import subprocess # noqa: F811 — function-local; except clauses reference it
|
||||
path = args.get("path", ".")
|
||||
command = args.get("command")
|
||||
|
||||
@@ -1295,9 +1191,8 @@ def _format_code(args: dict) -> dict:
|
||||
language = args.get("language")
|
||||
|
||||
ok, msg, resolved = validate_write_path(path)
|
||||
if not ok or resolved is None:
|
||||
if not ok:
|
||||
return {"success": False, "error": msg}
|
||||
assert resolved is not None # narrowing for the type checker
|
||||
|
||||
if not language:
|
||||
language = _detect_language(str(resolved))
|
||||
@@ -1590,64 +1485,6 @@ def _diagnose_behavior(args: dict) -> dict:
|
||||
return {"success": False, "error": f"Diagnosis failed: {e}"}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# P1-01 (#24): find_capability — the capability escape hatch
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
FIND_CAPABILITY_TOOL = "find_capability"
|
||||
|
||||
|
||||
def _find_capability(args: dict) -> dict:
|
||||
"""Execute the escape hatch: query the broker for matching capabilities.
|
||||
|
||||
Returns matching capability summaries (name + description + argument
|
||||
schema) so the model can call a discovered tool in the same session.
|
||||
The runtime auto-exposes returned native tools for the rest of the
|
||||
run (see local_tool_runtime). Degrade-not-crash: a broker failure is
|
||||
a tool error, never a crash.
|
||||
"""
|
||||
query = args.get("query", "")
|
||||
if not isinstance(query, str) or not query.strip():
|
||||
return {"success": False, "error": "Query is required — describe the capability you need"}
|
||||
|
||||
try:
|
||||
try:
|
||||
from rune import capability_broker as broker
|
||||
except ImportError: # direct script execution
|
||||
import capability_broker as broker
|
||||
matches = broker.find_capability(query)
|
||||
except Exception as exc:
|
||||
return {"success": False, "error": f"find_capability failed: {exc}"}
|
||||
|
||||
if not matches:
|
||||
return {
|
||||
"success": True,
|
||||
"result": (
|
||||
f"No capability matches {query!r}. The tools listed in your "
|
||||
"system prompt are all that is installed. Use "
|
||||
f"{FIND_CAPABILITY_TOOL} again with a broader description, "
|
||||
"or work with the tools you have."
|
||||
),
|
||||
"capabilities": [],
|
||||
"exposed": [],
|
||||
}
|
||||
|
||||
lines = []
|
||||
for match in matches:
|
||||
properties = match.get("parameters", {}).get("properties", {})
|
||||
arg_names = ", ".join(properties) if properties else "no arguments"
|
||||
lines.append(f"- {match['name']} ({match.get('risk', 'read-only')}): {match['description']} Args: {arg_names}")
|
||||
return {
|
||||
"success": True,
|
||||
"result": (
|
||||
"Matching capabilities (callable now in this session):\n"
|
||||
+ "\n".join(lines)
|
||||
),
|
||||
"capabilities": matches,
|
||||
"exposed": [match["name"] for match in matches],
|
||||
}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# AGENT LOOP — model → tool → result → model
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
+49
-123
@@ -24,112 +24,63 @@ import memory_env
|
||||
|
||||
VALID_CSS_PROPERTIES = {
|
||||
# Layout & sizing
|
||||
"align",
|
||||
"box-sizing",
|
||||
"content-align",
|
||||
"display",
|
||||
"dock",
|
||||
"height",
|
||||
"layer",
|
||||
"layers",
|
||||
"layout",
|
||||
"margin",
|
||||
"max-height",
|
||||
"max-width",
|
||||
"min-height",
|
||||
"min-width",
|
||||
"offset",
|
||||
"overflow",
|
||||
"padding",
|
||||
"position",
|
||||
"scrollbar-gutter",
|
||||
"scrollbar-size",
|
||||
"scrollbar-visibility",
|
||||
"width",
|
||||
"align", "box-sizing", "content-align", "display", "dock", "height",
|
||||
"layer", "layers", "layout", "margin", "max-height", "max-width",
|
||||
"min-height", "min-width", "offset", "overflow", "padding", "position",
|
||||
"scrollbar-gutter", "scrollbar-size", "scrollbar-visibility", "width",
|
||||
|
||||
# Colors & appearance
|
||||
"background",
|
||||
"background-tint",
|
||||
"color",
|
||||
"opacity",
|
||||
"tint",
|
||||
# Border (incl. per-side — verified valid Textual 8.x CSS: border-left etc.)
|
||||
"border",
|
||||
"border-left",
|
||||
"border-right",
|
||||
"border-top",
|
||||
"border-bottom",
|
||||
"border-subtitle-align",
|
||||
"border-subtitle-background",
|
||||
"border-subtitle-color",
|
||||
"border-subtitle-style",
|
||||
"border-title-align",
|
||||
"border-title-background",
|
||||
"border-title-color",
|
||||
"border-title-style",
|
||||
# Outline (incl. per-side)
|
||||
"background", "background-tint", "color", "opacity", "tint",
|
||||
|
||||
# Border
|
||||
"border", "border-subtitle-align", "border-subtitle-background",
|
||||
"border-subtitle-color", "border-subtitle-style",
|
||||
"border-title-align", "border-title-background",
|
||||
"border-title-color", "border-title-style",
|
||||
|
||||
# Outline
|
||||
"outline",
|
||||
"outline-left",
|
||||
"outline-right",
|
||||
"outline-top",
|
||||
"outline-bottom",
|
||||
|
||||
# Text
|
||||
"text-align",
|
||||
"text-opacity",
|
||||
"text-overflow",
|
||||
"text-style",
|
||||
"text-wrap",
|
||||
"text-align", "text-opacity", "text-overflow", "text-style", "text-wrap",
|
||||
|
||||
# Grid
|
||||
"column-span",
|
||||
"grid-columns",
|
||||
"grid-gutter",
|
||||
"grid-rows",
|
||||
"grid-size",
|
||||
"column-span", "grid-columns", "grid-gutter", "grid-rows", "grid-size",
|
||||
"row-span",
|
||||
|
||||
# Hatch
|
||||
"hatch",
|
||||
|
||||
# Keyline
|
||||
"keyline",
|
||||
|
||||
# Links
|
||||
"link-background",
|
||||
"link-background-hover",
|
||||
"link-color",
|
||||
"link-color-hover",
|
||||
"link-style",
|
||||
"link-style-hover",
|
||||
"link-background", "link-background-hover", "link-color",
|
||||
"link-color-hover", "link-style", "link-style-hover",
|
||||
|
||||
# Pointer
|
||||
"pointer",
|
||||
|
||||
# Scrollbar colors
|
||||
"scrollbar-background",
|
||||
"scrollbar-background-active",
|
||||
"scrollbar-background-hover",
|
||||
"scrollbar-color",
|
||||
"scrollbar-color-active",
|
||||
"scrollbar-color-hover",
|
||||
"scrollbar-background", "scrollbar-background-active",
|
||||
"scrollbar-background-hover", "scrollbar-color",
|
||||
"scrollbar-color-active", "scrollbar-color-hover",
|
||||
"scrollbar-corner-color",
|
||||
|
||||
# Visibility
|
||||
"visibility",
|
||||
}
|
||||
|
||||
# Pseudo-classes that are valid in Textual CSS
|
||||
VALID_PSEUDO_CLASSES = {
|
||||
"focus",
|
||||
"hover",
|
||||
"disabled",
|
||||
"enabled",
|
||||
"checked",
|
||||
"selected",
|
||||
"dark",
|
||||
"light",
|
||||
"inline",
|
||||
"ansi",
|
||||
"not-ansi",
|
||||
"focus", "hover", "disabled", "enabled", "checked", "selected",
|
||||
"dark", "light", "inline", "ansi", "not-ansi",
|
||||
}
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# CSS PARSING
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def extract_css_properties(css_text: str) -> list[tuple[str, int, str]]:
|
||||
"""Extract property names from CSS text.
|
||||
|
||||
@@ -137,15 +88,15 @@ def extract_css_properties(css_text: str) -> list[tuple[str, int, str]]:
|
||||
"""
|
||||
results = []
|
||||
# Remove comments
|
||||
css_text = re.sub(r"/\*.*?\*/", "", css_text, flags=re.DOTALL)
|
||||
css_text = re.sub(r'/\*.*?\*/', '', css_text, flags=re.DOTALL)
|
||||
|
||||
for i, line in enumerate(css_text.split("\n"), 1):
|
||||
for i, line in enumerate(css_text.split('\n'), 1):
|
||||
line = line.strip()
|
||||
# Skip selectors, comments, empty lines
|
||||
if not line or line.startswith(("/*", "*", "#", ".", "&", "@", "{", "}")):
|
||||
if not line or line.startswith(('/*', '*', '#', '.', '&', '@', '{', '}')):
|
||||
continue
|
||||
# Match property: value patterns
|
||||
match = re.match(r"^([a-z][a-z0-9-]*)\s*:", line)
|
||||
match = re.match(r'^([a-z][a-z0-9-]*)\s*:', line)
|
||||
if match:
|
||||
prop = match.group(1)
|
||||
results.append((prop, i, line))
|
||||
@@ -192,13 +143,11 @@ def _edit_distance(a: str, b: str) -> int:
|
||||
curr = [i + 1]
|
||||
for j, cb in enumerate(b):
|
||||
cost = 0 if ca == cb else 1
|
||||
curr.append(
|
||||
min(
|
||||
curr[j] + 1, # insert
|
||||
prev[j + 1] + 1, # delete
|
||||
prev[j] + cost, # replace
|
||||
)
|
||||
)
|
||||
curr.append(min(
|
||||
curr[j] + 1, # insert
|
||||
prev[j + 1] + 1, # delete
|
||||
prev[j] + cost, # replace
|
||||
))
|
||||
prev = curr
|
||||
return prev[-1]
|
||||
|
||||
@@ -207,7 +156,6 @@ def _edit_distance(a: str, b: str) -> int:
|
||||
# MODULE VALIDATION
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def validate_module_css(module_path: str) -> list[str]:
|
||||
"""Validate all DEFAULT_CSS and CSS class variables in a Python module.
|
||||
|
||||
@@ -225,7 +173,7 @@ def validate_module_css(module_path: str) -> list[str]:
|
||||
for match in re.finditer(css_pattern, source, re.DOTALL):
|
||||
css_text = match.group(1)
|
||||
# Find line number
|
||||
line_num = source[: match.start()].count("\n") + 1
|
||||
line_num = source[:match.start()].count('\n') + 1
|
||||
module_name = os.path.basename(module_path)
|
||||
css_errors = validate_css(css_text, f"{module_name}:L{line_num}")
|
||||
errors.extend(css_errors)
|
||||
@@ -261,7 +209,6 @@ def validate_all_modules(rune_dir: str) -> list[str]:
|
||||
# IMPORT VALIDATION
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def validate_imports(rune_dir: str) -> list[str]:
|
||||
"""Validate that all imports in rune_tui.py resolve correctly.
|
||||
|
||||
@@ -278,32 +225,17 @@ def validate_imports(rune_dir: str) -> list[str]:
|
||||
return [f"Could not read rune_tui.py: {e}"]
|
||||
|
||||
# Find all import statements
|
||||
import_pattern = r"^(?:from\s+(\S+)\s+import|import\s+(\S+))"
|
||||
import_pattern = r'^(?:from\s+(\S+)\s+import|import\s+(\S+))'
|
||||
for match in re.finditer(import_pattern, source, re.MULTILINE):
|
||||
module_name = match.group(1) or match.group(2)
|
||||
line_num = source[: match.start()].count("\n") + 1
|
||||
line_num = source[:match.start()].count('\n') + 1
|
||||
|
||||
# Skip stdlib and known third-party
|
||||
if module_name in (
|
||||
"os",
|
||||
"sys",
|
||||
"json",
|
||||
"re",
|
||||
"time",
|
||||
"asyncio",
|
||||
"threading",
|
||||
"hashlib",
|
||||
"sqlite3",
|
||||
"urllib",
|
||||
"dataclasses",
|
||||
"typing",
|
||||
"datetime",
|
||||
"textual",
|
||||
"textual_autocomplete",
|
||||
"rich",
|
||||
"logging",
|
||||
"difflib",
|
||||
):
|
||||
if module_name in ("os", "sys", "json", "re", "time", "asyncio",
|
||||
"threading", "hashlib", "sqlite3", "urllib",
|
||||
"dataclasses", "typing", "datetime", "textual",
|
||||
"textual_autocomplete", "rich", "logging",
|
||||
"difflib"):
|
||||
continue
|
||||
|
||||
# Check if local module exists
|
||||
@@ -318,9 +250,7 @@ def validate_imports(rune_dir: str) -> list[str]:
|
||||
scripts_dir = memory_env.scripts_dir()
|
||||
mod_file = os.path.join(scripts_dir, module_name + ".py")
|
||||
if not os.path.exists(mod_file):
|
||||
errors.append(
|
||||
f"rune_tui.py:{line_num}: Import '{module_name}' — module not found in rune/ or scripts/"
|
||||
)
|
||||
errors.append(f"rune_tui.py:{line_num}: Import '{module_name}' — module not found in rune/ or scripts/")
|
||||
|
||||
return errors
|
||||
|
||||
@@ -329,7 +259,6 @@ def validate_imports(rune_dir: str) -> list[str]:
|
||||
# WIDGET COMPOSITION VALIDATION
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def validate_composition(rune_dir: str) -> list[str]:
|
||||
"""Validate widget composition patterns.
|
||||
|
||||
@@ -351,7 +280,6 @@ def validate_composition(rune_dir: str) -> list[str]:
|
||||
# FULL VALIDATION
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def run_full_validation(rune_dir: str = None) -> list[str]:
|
||||
"""Run all validation checks.
|
||||
|
||||
@@ -381,7 +309,6 @@ def run_full_validation(rune_dir: str = None) -> list[str]:
|
||||
# SELF-TEST
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def self_test():
|
||||
"""Run self-tests for the validator."""
|
||||
print("Running TUI Validator self-tests...")
|
||||
@@ -456,7 +383,6 @@ def self_test():
|
||||
# CLI
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: tui_validator.py [check|css|test]")
|
||||
|
||||
@@ -1,408 +0,0 @@
|
||||
"""Failure auto-filer (#122): subsystem failures become forge issues.
|
||||
|
||||
Core tests: signature extraction, dedup via NEW server-side search_issues,
|
||||
file-once + comment-on-repeat, the ONE tier gate (T0/T1 file, T2 journal
|
||||
only), and the spool contract (any client exception lands in the journal
|
||||
as a failure_auto_file_spool line, nothing ever raises into the failure
|
||||
path, T2 failures produce ZERO client calls).
|
||||
|
||||
``FailureAutoFiler.file(...)`` is the synchronous pipeline step (used by
|
||||
the worker thread, reconcile, and these tests); ``submit(...)`` is the
|
||||
failure-path entry that only enqueues.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from rune import event_bus
|
||||
from rune.failure_auto_filer import (
|
||||
FailureAutoFiler,
|
||||
JsonlSpool,
|
||||
error_class_of,
|
||||
excerpt,
|
||||
marker_for,
|
||||
redact,
|
||||
signature_of,
|
||||
)
|
||||
from rune.forge_client import ForgeClient, NullAdapter
|
||||
|
||||
MARKER = "[auto-failure:memory_selector:RuntimeError]"
|
||||
|
||||
|
||||
class FakeJournal:
|
||||
"""In-memory stand-in for the spool-journal protocol."""
|
||||
|
||||
def __init__(self):
|
||||
self.lines: list[dict] = []
|
||||
self.consumed: set[str] = set()
|
||||
|
||||
def append(self, record: dict) -> None:
|
||||
self.lines.append(record)
|
||||
|
||||
def unconsumed(self) -> list[dict]:
|
||||
return [
|
||||
r
|
||||
for r in self.lines
|
||||
if r.get("kind") == "failure_auto_file_spool" and r.get("spool_id") not in self.consumed
|
||||
]
|
||||
|
||||
def mark_consumed(self, spool_id: str) -> None:
|
||||
self.consumed.add(spool_id)
|
||||
|
||||
|
||||
class FakeForge:
|
||||
"""Test double recording every client call the filer makes.
|
||||
|
||||
search_issues mirrors the REAL repo-scoped list endpoint (fix #122
|
||||
gate): server q= filters by marker-in-TITLE only (like real Gitea q),
|
||||
NOT by mirroring the client's filter — the old fake filtered the same
|
||||
way the filer did, making the filer's own check untestable.
|
||||
"""
|
||||
|
||||
def __init__(self, search_results=None, fail_with: Exception | None = None):
|
||||
self.search_results = search_results if search_results is not None else []
|
||||
self.fail_with = fail_with
|
||||
self._next_number = 7
|
||||
self.created: list[dict] = []
|
||||
self.commented: list[dict] = []
|
||||
self.searched: list[str] = []
|
||||
|
||||
def search_issues(self, query: str) -> list[dict]:
|
||||
self.searched.append(query)
|
||||
if self.fail_with is not None:
|
||||
raise self.fail_with
|
||||
return [i for i in self.search_results if query in str(i.get("title", ""))]
|
||||
|
||||
def create_issue(self, title: str, body: str) -> dict:
|
||||
if self.fail_with is not None:
|
||||
raise self.fail_with
|
||||
self.created.append({"title": title, "body": body})
|
||||
issue = {"number": self._next_number, "title": title, "state": "open"}
|
||||
self._next_number += 1
|
||||
self.search_results.append(dict(issue)) # findable by later searches
|
||||
return issue
|
||||
|
||||
def add_comment(self, issue_number: int, body: str) -> dict:
|
||||
if self.fail_with is not None:
|
||||
raise self.fail_with
|
||||
self.commented.append({"issue_number": issue_number, "body": body})
|
||||
return {"created": True}
|
||||
|
||||
def get_issue_state(self, issue_number: int) -> dict:
|
||||
return {"state": "open"}
|
||||
|
||||
def list_open_issues(self) -> list[dict]:
|
||||
return []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _real_journal(tmp_path, monkeypatch):
|
||||
"""Point the canonical journal at tmp so emit() never touches real data."""
|
||||
monkeypatch.setattr("rune.event_bus._store_path", lambda: tmp_path / "events.jsonl")
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- signature
|
||||
|
||||
|
||||
def test_signature_is_subsystem_and_error_class():
|
||||
assert signature_of("memory_selector", "RuntimeError: boom") == ("memory_selector", "RuntimeError")
|
||||
|
||||
|
||||
def test_signature_from_plain_error_string_takes_first_token():
|
||||
assert signature_of("cron", "Connection refused to forge") == ("cron", "Connection")
|
||||
|
||||
|
||||
def test_signature_accepts_exception_instance():
|
||||
try:
|
||||
raise ValueError("bad payload")
|
||||
except ValueError as exc:
|
||||
assert signature_of("persona", exc) == ("persona", "ValueError")
|
||||
|
||||
|
||||
def test_error_class_of_exception_and_string():
|
||||
assert error_class_of(RuntimeError("x")) == "RuntimeError"
|
||||
assert error_class_of("TimeoutError: read timed out") == "TimeoutError"
|
||||
assert error_class_of("connection reset by peer") == "connection"
|
||||
|
||||
|
||||
def test_error_class_of_caps_single_token_at_64_chars():
|
||||
# #122 gate: a 10KB single-token error must not make a 10KB marker.
|
||||
huge = "E" * 10000
|
||||
capped = error_class_of(f"{huge}: boom")
|
||||
assert len(capped) == 64
|
||||
assert len(marker_for("subsystem", capped)) < 100 # marker stays bounded
|
||||
assert error_class_of("RuntimeError: ok") == "RuntimeError" # short: untouched
|
||||
|
||||
|
||||
def test_marker_format():
|
||||
assert marker_for("memory_selector", "RuntimeError") == MARKER
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- file path
|
||||
|
||||
|
||||
def test_first_occurrence_creates_issue_with_marker_title():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
assert len(forge.created) == 1
|
||||
assert forge.created[0]["title"].startswith(MARKER)
|
||||
assert forge.commented == []
|
||||
|
||||
|
||||
def test_repeat_occurrence_comments_on_existing_issue():
|
||||
forge = FakeForge(search_results=[{"number": 7, "title": MARKER + " memory_selector"}])
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
assert forge.created == [] # dedup: never a second issue per signature
|
||||
assert len(forge.commented) == 1
|
||||
assert "+1 occurrences" in forge.commented[0]["body"]
|
||||
assert "RuntimeError" in forge.commented[0]["body"]
|
||||
|
||||
|
||||
def test_each_repeat_comments_never_creates():
|
||||
forge = FakeForge(search_results=[{"number": 7, "title": MARKER + " memory_selector"}])
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
filer.file("memory_selector", "RuntimeError: another payload", tier=1)
|
||||
assert forge.created == []
|
||||
assert len(forge.commented) == 2
|
||||
assert "another payload" in forge.commented[1]["body"]
|
||||
|
||||
|
||||
def test_search_results_without_marker_in_title_are_ignored():
|
||||
# Server-side q= is fuzzy (title OR body); only a title match dedups.
|
||||
forge = FakeForge(search_results=[{"number": 9, "title": "unrelated discussion of the marker"}])
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
assert len(forge.created) == 1
|
||||
assert forge.commented == []
|
||||
|
||||
|
||||
def test_tier_gate_t2_produces_zero_client_calls():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal())
|
||||
filer.file("cron", "Connection refused", tier=2)
|
||||
assert forge.searched == []
|
||||
assert forge.created == []
|
||||
assert forge.commented == []
|
||||
|
||||
|
||||
def test_tier_gate_t0_and_t1_file():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
|
||||
filer.file("memory_selector", "RuntimeError: a", tier=0)
|
||||
filer.file("warm_router", "RuntimeError: b", tier=1)
|
||||
assert len(forge.created) == 2
|
||||
|
||||
|
||||
def test_create_issue_body_carries_tier_and_count():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal())
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
body = forge.created[0]["body"]
|
||||
assert "tier=1" in body
|
||||
assert "occurrences=1" in body
|
||||
|
||||
|
||||
def test_error_excerpt_is_truncated():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal())
|
||||
filer.file("memory_selector", "RuntimeError: " + ("x" * 500), tier=1)
|
||||
body = forge.created[0]["body"]
|
||||
assert ("x" * 500) not in body
|
||||
assert ("x" * 100) in body
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- redaction
|
||||
|
||||
|
||||
def test_redact_strips_query_string_credentials():
|
||||
assert (
|
||||
redact("GET https://forge.example.com/api?token=supersecret&page=2 failed")
|
||||
== "GET https://forge.example.com/api?token=<redacted>&page=2 failed"
|
||||
)
|
||||
assert "password=hunter2" not in redact("postgres://host/db?password=hunter2&x=1")
|
||||
assert "api_key=abcd" not in redact("https://api.example.com/v1?api_key=abcd&y=2")
|
||||
assert "key=zzz" not in redact("https://x.example/?key=zzz")
|
||||
|
||||
|
||||
def test_redact_strips_url_basic_auth_and_dsn_credentials():
|
||||
out = redact("connect failed for postgres://alice:hunter2@db.example.com:5432/rune")
|
||||
assert "hunter2" not in out
|
||||
assert "alice" not in out
|
||||
assert "postgres://<redacted>:<redacted>@db.example.com:5432/rune" in out
|
||||
out2 = redact("redis://:s3cr3t@cache.example.com:6379/0")
|
||||
assert "s3cr3t" not in out2
|
||||
|
||||
|
||||
def test_redact_strips_bearer_and_assignment_credentials():
|
||||
out = redact("Authorization: Bearer eyJhbGciOi.eyJzdWIi.SflKxwRJSMeK")
|
||||
assert "eyJhbGciOi" not in out
|
||||
assert "<redacted>" in out
|
||||
out2 = redact("login failed with password='hunter2' and secret=abcdef1234")
|
||||
assert "hunter2" not in out2
|
||||
assert "abcdef1234" not in out2
|
||||
|
||||
|
||||
def test_redact_keeps_error_class_and_shape_visible():
|
||||
out = redact("RuntimeError: connection to https://user:pass@forge.example.com refused (port 443)")
|
||||
assert out.startswith("RuntimeError: ")
|
||||
assert "refused (port 443)" in out # non-credential detail survives
|
||||
# Redaction is deterministic on the credential span only:
|
||||
body = "connection to https://user:pass@forge.example.com refused (port 443)"
|
||||
assert out == "RuntimeError: " + redact(body)
|
||||
assert redact("plain error, nothing sensitive") == "plain error, nothing sensitive"
|
||||
|
||||
|
||||
def test_excerpt_redacts_credentials():
|
||||
# excerpt() is what lands in persistent forge issues and spool lines:
|
||||
# it must redact before returning.
|
||||
assert "hunter2" not in excerpt("postgres://bob:hunter2@db/rune down")
|
||||
assert "<redacted>" in excerpt("GET https://forge.example.com/api?token=topsecret")
|
||||
|
||||
|
||||
def test_credentialed_error_never_lands_raw_in_issue_or_spool():
|
||||
forge = FakeForge(fail_with=ConnectionError("connect: https://user:hunter2@forge.example.com/api?token=t0psecret"))
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1)
|
||||
filer.file("memory_selector", "RuntimeError: dsn postgres://alice:s3cr3t@db.example.com/rune", tier=1)
|
||||
spooled = journal.unconsumed()[0]
|
||||
for blob in (json.dumps(forge.created), json.dumps(spooled)):
|
||||
for secret in ("hunter2", "s3cr3t", "t0psecret"):
|
||||
assert secret not in blob
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- spool path
|
||||
|
||||
|
||||
def test_client_exception_spools_and_never_raises():
|
||||
forge = FakeForge(fail_with=ConnectionError("forge down"))
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal)
|
||||
# Must NOT raise.
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
pending = journal.unconsumed()
|
||||
assert len(pending) == 1
|
||||
line = pending[0]
|
||||
assert line["kind"] == "failure_auto_file_spool"
|
||||
assert line["signature"] == ["memory_selector", "RuntimeError"]
|
||||
assert line["tier"] == 1
|
||||
assert "boom" in line["error"]
|
||||
assert "timestamp" in line
|
||||
assert forge.created == []
|
||||
|
||||
|
||||
def test_spool_write_failure_still_never_raises():
|
||||
forge = FakeForge()
|
||||
journal = FakeJournal()
|
||||
|
||||
def explode(record):
|
||||
raise OSError("disk full")
|
||||
|
||||
journal.append = explode
|
||||
filer = FailureAutoFiler(forge, journal)
|
||||
# Neither forge failure nor journal failure may escape.
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
|
||||
|
||||
def test_jsonl_spool_line_is_durable_and_json(tmp_path):
|
||||
path = tmp_path / "events.jsonl"
|
||||
forge = FakeForge(fail_with=ConnectionError("forge down"))
|
||||
spool = JsonlSpool(path)
|
||||
filer = FailureAutoFiler(forge, spool)
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
rows = [json.loads(line) for line in path.read_text().splitlines()]
|
||||
assert rows and rows[-1]["kind"] == "failure_auto_file_spool"
|
||||
assert spool.unconsumed()[0]["spool_id"] == rows[-1]["spool_id"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- protocol
|
||||
|
||||
|
||||
def test_null_adapter_search_returns_empty():
|
||||
assert NullAdapter().search_issues("anything") == []
|
||||
|
||||
|
||||
def test_filer_accepts_protocol_clients():
|
||||
# Structural check: the fake satisfies ForgeClient incl. search_issues.
|
||||
assert issubclass(FakeForge, ForgeClient)
|
||||
assert issubclass(NullAdapter, ForgeClient)
|
||||
|
||||
|
||||
# ------------------------------------------------- queue / worker / kill tests
|
||||
|
||||
|
||||
def test_submit_enqueues_and_drain_delivers_writes():
|
||||
forge = FakeForge(search_results=[{"number": 7, "title": MARKER + " memory_selector"}])
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=1)
|
||||
filer.submit("cron", "Connection refused", tier=2) # gated: zero client calls
|
||||
filer.submit("memory_selector", "RuntimeError: a", tier=1)
|
||||
filer.submit("memory_selector", "RuntimeError: b", tier=1)
|
||||
filer.drain(timeout=5)
|
||||
assert forge.created == [] # existing issue deduped
|
||||
# Worker may batch both queued occurrences into one comment, or handle
|
||||
# them separately — either way: comments only, never a second issue.
|
||||
assert 1 <= len(forge.commented) <= 2
|
||||
|
||||
|
||||
def test_queue_is_bounded_and_overflow_drops_spools_and_logs(caplog):
|
||||
# #122 gate: the queue must be bounded (a forge outage must not balloon
|
||||
# memory) and an overflow occurrence is DROPPED from the queue but
|
||||
# SPOOLED to the journal (nothing lost) + logged — never blocked.
|
||||
import logging
|
||||
|
||||
forge = FakeForge()
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, wait=True)
|
||||
assert filer._queue.maxsize == 1000
|
||||
for i in range(1000):
|
||||
filer.submit("memory_selector", f"RuntimeError: fill {i}", tier=1)
|
||||
assert filer._queue.full()
|
||||
with caplog.at_level(logging.WARNING, logger="rune.failure_auto_filer"):
|
||||
filer.submit("memory_selector", "RuntimeError: dropped-occurrence", tier=1)
|
||||
assert filer._queue.full() # queue did not grow past maxsize
|
||||
pending = journal.unconsumed()
|
||||
assert any(r["error"].endswith("dropped-occurrence") for r in pending) # spooled
|
||||
assert any("queue full" in rec.getMessage() for rec in caplog.records) # logged
|
||||
|
||||
|
||||
def test_worker_exception_never_propagates():
|
||||
forge = FakeForge(fail_with=ConnectionError("forge down"))
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal)
|
||||
filer.submit("memory_selector", "RuntimeError: boom", tier=1)
|
||||
filer.drain(timeout=5) # worker hit the exception; must not raise here
|
||||
assert len(journal.unconsumed()) == 1
|
||||
|
||||
|
||||
def test_t2_kill_test_zero_client_calls_through_worker():
|
||||
# Frozen kill test: a T2 failure must produce ZERO client calls even
|
||||
# when the worker thread drains the queue.
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal())
|
||||
filer.submit("cron", "Connection refused", tier=2)
|
||||
filer.submit("cron", "JSONDecodeError while parsing", tier=2)
|
||||
filer.drain(timeout=5)
|
||||
assert forge.searched == []
|
||||
assert forge.created == []
|
||||
assert forge.commented == []
|
||||
|
||||
|
||||
def test_forge_down_kill_test_never_raises_into_failure_path():
|
||||
forge = FakeForge(fail_with=ConnectionError("forge down"))
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal)
|
||||
try:
|
||||
filer.submit("memory_selector", "RuntimeError: boom", tier=1)
|
||||
filer.drain(timeout=5)
|
||||
except Exception as exc: # pragma: no cover - the assertion below bites
|
||||
pytest.fail(f"filer raised into the failure path: {exc!r}")
|
||||
assert len(journal.unconsumed()) == 1
|
||||
|
||||
|
||||
def test_event_bus_emit_unchanged_after_filer_import():
|
||||
# Spool rides the canonical journal; regular emit keeps working.
|
||||
event_bus.emit("user.message", source="t", payload={"k": "v"})
|
||||
@@ -1,416 +0,0 @@
|
||||
"""#122 Task 2: burst coalescing + spool reconciliation.
|
||||
|
||||
Burst coalescing: after the first occurrence files, subsequent occurrences
|
||||
within the dedup window batch into ONE comment per batch_size occurrences
|
||||
or 10 minutes (whichever first) — a 3am crash-loop must not write 200
|
||||
comments. The worker additionally coalesces everything already queued
|
||||
into one write per signature. Reconcile: replay unconsumed spool lines
|
||||
through the same pipeline (dedup re-applies via search_issues); filed
|
||||
lines are marked consumed; forge still down → spool preserved, never
|
||||
raises.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from rune.failure_auto_filer import (
|
||||
BATCH_INTERVAL_SECONDS,
|
||||
FailureAutoFiler,
|
||||
JsonlSpool,
|
||||
)
|
||||
from rune.forge_client import NullAdapter
|
||||
|
||||
from tests.test_failure_auto_filer import FakeForge, FakeJournal, MARKER
|
||||
|
||||
|
||||
# ------------------------------------------------------------ burst coalescing
|
||||
|
||||
|
||||
def test_burst_of_50_produces_at_most_2_client_writes():
|
||||
# First occurrence files; everything already queued coalesces into one
|
||||
# comment; the tail (< batch_size) stays pending: 1 create + ≤1 comment.
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=10, wait=True)
|
||||
for i in range(50):
|
||||
filer.submit("memory_selector", f"RuntimeError: boom {i}", tier=1)
|
||||
filer.drain(timeout=5) # the whole backlog coalesces into ONE write
|
||||
assert len(forge.created) == 1
|
||||
assert len(forge.created) + len(forge.commented) <= 2
|
||||
|
||||
|
||||
def test_batch_flushes_every_batch_size_occurrences():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=10, wait=False)
|
||||
for i in range(25):
|
||||
filer.file("memory_selector", f"RuntimeError: boom {i}", tier=1)
|
||||
assert len(forge.created) == 1
|
||||
# Comments when pending EXCEEDS the batch size (occurrences 11 and 21):
|
||||
# one comment per 10 occurrences, tail pending.
|
||||
assert len(forge.commented) == 2
|
||||
assert "+10 occurrences" in forge.commented[0]["body"]
|
||||
assert "+20 occurrences" in forge.commented[1]["body"]
|
||||
|
||||
|
||||
def test_batch_flushes_after_ten_minutes():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=10, wait=False)
|
||||
start = time.monotonic()
|
||||
filer.file("memory_selector", "RuntimeError: boom 0", tier=1, _now=start) # files
|
||||
later = start + BATCH_INTERVAL_SECONDS + 1 # 10 minutes pass
|
||||
filer.file("memory_selector", "RuntimeError: boom 1", tier=1, _now=later)
|
||||
assert len(forge.created) == 1
|
||||
assert len(forge.commented) == 1 # window expired -> flush on next occurrence
|
||||
assert "+1 occurrences" in forge.commented[0]["body"]
|
||||
|
||||
|
||||
def test_burst_counter_is_per_signature():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=10, wait=False)
|
||||
for i in range(12):
|
||||
filer.file("memory_selector", f"RuntimeError: boom {i}", tier=1)
|
||||
for i in range(3):
|
||||
filer.file("warm_router", f"TimeoutError: slow {i}", tier=1)
|
||||
assert len(forge.created) == 2 # one per signature
|
||||
assert len(forge.commented) == 1 # memory_selector hit 10; warm_router (3) not yet
|
||||
|
||||
|
||||
def test_burst_tail_below_batch_size_stays_pending_until_window():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=10, wait=False)
|
||||
for i in range(7):
|
||||
filer.file("memory_selector", f"RuntimeError: boom {i}", tier=1)
|
||||
filer.drain(timeout=5)
|
||||
assert len(forge.created) == 1
|
||||
assert len(forge.commented) == 0 # 7 < batch_size, window not expired
|
||||
|
||||
|
||||
def test_burst_batched_comment_carries_last_error_excerpt():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=10, wait=True)
|
||||
for i in range(11): # 11th occurrence crosses the batch threshold
|
||||
filer.file("memory_selector", f"RuntimeError: boom {i}", tier=1)
|
||||
filer.drain(timeout=5)
|
||||
assert len(forge.commented) == 1
|
||||
assert "+10 occurrences" in forge.commented[0]["body"]
|
||||
assert "boom 10" in forge.commented[0]["body"] # last error of the batch
|
||||
|
||||
|
||||
def test_window_resets_after_flush():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), batch_size=10, wait=False)
|
||||
base = time.monotonic()
|
||||
filer.file("memory_selector", "RuntimeError: a0", tier=1, _now=base) # files
|
||||
for i in range(1, 12):
|
||||
filer.file("memory_selector", f"RuntimeError: a{i}", tier=1, _now=base + i * 0.1)
|
||||
assert len(forge.commented) == 1 # flush at pending=11 (> batch_size)
|
||||
# The window restarted at that flush: no second comment for 3 more.
|
||||
for i in range(3):
|
||||
filer.file("memory_selector", f"RuntimeError: b{i}", tier=1, _now=base + 2 + i * 0.1)
|
||||
assert len(forge.commented) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- reconcile
|
||||
|
||||
|
||||
def test_reconcile_groups_spool_lines_by_signature_into_one_write():
|
||||
# #122 gate: 10 spooled lines of ONE signature (a long outage) must
|
||||
# produce exactly ONE client write on reconcile — not a comment flood.
|
||||
forge = FakeForge()
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1, wait=False)
|
||||
for i in range(10):
|
||||
journal.append(
|
||||
{
|
||||
"kind": "failure_auto_file_spool",
|
||||
"spool_id": f"spool-{i}",
|
||||
"timestamp": "t",
|
||||
"signature": ["memory_selector", "RuntimeError"],
|
||||
"tier": 1,
|
||||
"count": 1,
|
||||
"error": f"RuntimeError: boom {i}",
|
||||
}
|
||||
)
|
||||
report = filer.reconcile()
|
||||
assert len(forge.created) + len(forge.commented) == 1 # ONE write
|
||||
assert len(forge.created) == 1 # dedup still applies: issue created once
|
||||
assert "occurrences=10" in forge.created[0]["body"] # aggregated count
|
||||
assert "boom 9" in forge.created[0]["body"] # last error of the group
|
||||
assert report["filed"] == 10
|
||||
assert journal.unconsumed() == []
|
||||
|
||||
|
||||
def test_reconcile_group_comments_once_with_offline_count():
|
||||
# Same grouping when the issue already exists: ONE aggregated comment.
|
||||
forge = FakeForge(search_results=[{"number": 7, "title": MARKER + " memory_selector"}])
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1, wait=False)
|
||||
for i in range(4):
|
||||
journal.append(
|
||||
{
|
||||
"kind": "failure_auto_file_spool",
|
||||
"spool_id": f"spool-{i}",
|
||||
"timestamp": "t",
|
||||
"signature": ["memory_selector", "RuntimeError"],
|
||||
"tier": 1,
|
||||
"count": 1,
|
||||
"error": f"RuntimeError: boom {i}",
|
||||
}
|
||||
)
|
||||
report = filer.reconcile()
|
||||
assert forge.created == []
|
||||
assert len(forge.commented) == 1
|
||||
assert "+4 occurrences while offline" in forge.commented[0]["body"]
|
||||
assert "boom 3" in forge.commented[0]["body"]
|
||||
assert report["filed"] == 4
|
||||
assert journal.unconsumed() == []
|
||||
|
||||
|
||||
def test_reconcile_replays_spool_and_marks_consumed():
|
||||
forge = FakeForge()
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1, wait=False)
|
||||
forge.fail_with = ConnectionError("forge down")
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
assert len(journal.unconsumed()) == 1
|
||||
forge.fail_with = None
|
||||
filer.reconcile()
|
||||
assert len(forge.created) == 1
|
||||
assert forge.created[0]["title"].startswith(MARKER)
|
||||
assert journal.unconsumed() == [] # marked consumed
|
||||
|
||||
|
||||
def test_reconcile_dedups_against_live_issues():
|
||||
forge = FakeForge(search_results=[{"number": 7, "title": MARKER + " memory_selector"}])
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1, wait=False)
|
||||
forge.fail_with = ConnectionError("forge down")
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
forge.fail_with = None
|
||||
filer.reconcile()
|
||||
assert forge.created == [] # dedup re-applied: comment, not a second issue
|
||||
assert len(forge.commented) == 1
|
||||
assert journal.unconsumed() == []
|
||||
|
||||
|
||||
def test_reconcile_with_forge_still_down_preserves_spool():
|
||||
forge = FakeForge(fail_with=ConnectionError("forge still down"))
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1, wait=False)
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
pending_before = [dict(r) for r in journal.unconsumed()]
|
||||
filer.reconcile() # must not raise
|
||||
assert journal.unconsumed() == pending_before # preserved, still pending
|
||||
assert forge.created == []
|
||||
|
||||
|
||||
def test_reconcile_on_jsonl_spool_marks_consumed_in_sidecar(tmp_path):
|
||||
path = tmp_path / "events.jsonl"
|
||||
forge = FakeForge()
|
||||
spool = JsonlSpool(path)
|
||||
filer = FailureAutoFiler(forge, spool, batch_size=1, wait=False)
|
||||
forge.fail_with = ConnectionError("down")
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
assert len(spool.unconsumed()) == 1
|
||||
canonical_before = path.read_bytes()
|
||||
forge.fail_with = None
|
||||
filer.reconcile()
|
||||
assert spool.unconsumed() == []
|
||||
# Canonical journal untouched — consumption lives in the sidecar only.
|
||||
assert path.read_bytes() == canonical_before
|
||||
assert (tmp_path / "events.jsonl.consumed").exists()
|
||||
|
||||
|
||||
def _spool_record(spool_id: str) -> dict:
|
||||
return {
|
||||
"kind": "failure_auto_file_spool",
|
||||
"spool_id": spool_id,
|
||||
"timestamp": "t",
|
||||
"signature": ["s", "E"],
|
||||
"tier": 1,
|
||||
"count": 1,
|
||||
"error": "E: x",
|
||||
}
|
||||
|
||||
|
||||
def test_mark_consumed_never_rewrites_the_canonical_journal(tmp_path):
|
||||
# #122 gate: mark_consumed must be append-only. The canonical journal
|
||||
# is shared with DurableEventStore — a read-modify-write there is
|
||||
# racy and a crash mid-rewrite poisons the canonical store.
|
||||
path = tmp_path / "events.jsonl"
|
||||
spool = JsonlSpool(path)
|
||||
spool.append(_spool_record("spool-x"))
|
||||
before = path.read_bytes()
|
||||
spool.mark_consumed("spool-x")
|
||||
assert path.read_bytes() == before # byte-identical: no rewrite, ever
|
||||
assert spool.unconsumed() == [] # consumption recorded in the sidecar
|
||||
assert (tmp_path / "events.jsonl.consumed").read_text().splitlines() == ["spool-x"]
|
||||
|
||||
|
||||
def test_concurrent_append_between_read_and_mark_is_not_lost(tmp_path, monkeypatch):
|
||||
# A DurableEventStore.append landing while mark_consumed runs must
|
||||
# survive. The old implementation read the journal, then rewrote it —
|
||||
# the append landed in between and was silently DESTROYED.
|
||||
import builtins
|
||||
|
||||
path = tmp_path / "events.jsonl"
|
||||
spool = JsonlSpool(path)
|
||||
spool.append(_spool_record("spool-x"))
|
||||
real_open = builtins.open
|
||||
state = {"fired": False}
|
||||
|
||||
def racing_open(file, mode="r", *a, **k):
|
||||
# The concurrent O_APPEND lands while mark_consumed is mid-flight
|
||||
# (fires on the first write/append-mode open inside the mark).
|
||||
if not state["fired"] and any(m in mode for m in ("a", "w", "x")):
|
||||
state["fired"] = True
|
||||
with real_open(path, "a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps({"event_id": "e-race", "event_type": "task.failed"}) + "\n")
|
||||
return real_open(file, mode, *a, **k)
|
||||
|
||||
monkeypatch.setattr(builtins, "open", racing_open)
|
||||
spool.mark_consumed("spool-x")
|
||||
rows = [json.loads(line) for line in path.read_text().splitlines()]
|
||||
assert any(r.get("event_id") == "e-race" for r in rows) # NOT destroyed
|
||||
assert spool.unconsumed() == []
|
||||
|
||||
|
||||
def test_reconcile_routes_tier2_line_through_the_tier_gate():
|
||||
# #122 gate: a crafted tier=2 spool line must stay journal-only on
|
||||
# replay — the SAME one tier gate as the live path (no _write bypass).
|
||||
forge = FakeForge()
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, wait=False)
|
||||
journal.append(_spool_record("spool-t2") | {"tier": 2})
|
||||
report = filer.reconcile()
|
||||
assert forge.searched == [] # zero client calls: journal-only on replay
|
||||
assert forge.created == []
|
||||
assert forge.commented == []
|
||||
assert report["skipped"] == 1
|
||||
assert journal.unconsumed() == [] # resolved: consumed, never written
|
||||
|
||||
|
||||
def test_reconcile_mixed_group_tier2_lines_dropped_tier1_filed():
|
||||
# A group mixing tiers keeps filing the T0/T1 lines; only the T2
|
||||
# lines are gate-dropped (journal-only).
|
||||
forge = FakeForge()
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1, wait=False)
|
||||
journal.append(_spool_record("spool-t1") | {"tier": 1})
|
||||
journal.append(_spool_record("spool-t2") | {"tier": 2})
|
||||
report = filer.reconcile()
|
||||
assert len(forge.created) == 1
|
||||
assert report["filed"] == 1
|
||||
assert report["skipped"] == 1
|
||||
assert journal.unconsumed() == []
|
||||
|
||||
|
||||
def test_reconcile_and_worker_single_flight_per_signature():
|
||||
# #122 gate: the reconcile thread and the worker racing on the SAME
|
||||
# signature must not duplicate the issue — single-flight lets exactly
|
||||
# one write land per signature at a time.
|
||||
import threading
|
||||
|
||||
forge = FakeForge()
|
||||
entries = {"n": 0}
|
||||
release = threading.Event()
|
||||
|
||||
def slow_search(query):
|
||||
entries["n"] += 1
|
||||
if entries["n"] == 1:
|
||||
release.wait(timeout=5) # hold the first writer inside the window
|
||||
# Like a real forge: issues created earlier ARE found by later
|
||||
# searches, and search results carry the issue number.
|
||||
return [
|
||||
{"number": n, "title": i["title"]} for n, i in enumerate(forge.created, start=100) if query in i["title"]
|
||||
]
|
||||
|
||||
forge.search_issues = slow_search # type: ignore[method-assign]
|
||||
journal = FakeJournal()
|
||||
journal.append(_spool_record("spool-race"))
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1, wait=False)
|
||||
filer.submit("s", "E: x", tier=1) # WORKER path (submit), same signature
|
||||
deadline = time.monotonic() + 5
|
||||
while entries["n"] < 1 and time.monotonic() < deadline:
|
||||
time.sleep(0.01) # worker is inside the write window
|
||||
reconcile_report = filer.reconcile() # races the worker on the same sig
|
||||
release.set()
|
||||
filer.drain(timeout=5)
|
||||
assert len(forge.created) == 1 # NEVER a duplicate issue
|
||||
assert forge.commented == []
|
||||
assert len(forge.created) + len(forge.commented) == 1
|
||||
# The racing replay stayed spooled (not falsely consumed): the
|
||||
# in-flight writer owned the signature; a later replay dedups.
|
||||
assert reconcile_report["filed"] == 0
|
||||
filer.reconcile()
|
||||
assert len(forge.created) == 1 # dedup re-applies: comment, not issue 2
|
||||
assert len(forge.commented) == 1
|
||||
assert journal.unconsumed() == []
|
||||
|
||||
|
||||
def test_reconcile_empty_spool_is_a_noop():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), wait=False)
|
||||
filer.reconcile()
|
||||
assert forge.searched == []
|
||||
assert forge.created == []
|
||||
|
||||
|
||||
def test_reconcile_files_and_consumes_multiple_spooled_signatures():
|
||||
forge = FakeForge()
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(forge, journal, batch_size=1, wait=False)
|
||||
forge.fail_with = ConnectionError("down")
|
||||
filer.file("memory_selector", "RuntimeError: boom", tier=1)
|
||||
filer.file("warm_router", "TimeoutError: slow", tier=1)
|
||||
assert len(journal.unconsumed()) == 2
|
||||
forge.fail_with = None
|
||||
filer.reconcile()
|
||||
assert len(forge.created) == 2
|
||||
assert journal.unconsumed() == []
|
||||
|
||||
|
||||
def test_null_adapter_reconcile_consumes_spool():
|
||||
# NullAdapter never raises; a search returns [] and create_issue is a
|
||||
# no-op dict — the filer treats that as "nothing was filed" and keeps
|
||||
# the line spooled rather than falsely consuming it.
|
||||
journal = FakeJournal()
|
||||
filer = FailureAutoFiler(NullAdapter(), journal, batch_size=1, wait=False)
|
||||
journal.append(
|
||||
{
|
||||
"kind": "failure_auto_file_spool",
|
||||
"spool_id": "spool-1",
|
||||
"timestamp": "t",
|
||||
"signature": ["memory_selector", "RuntimeError"],
|
||||
"tier": 1,
|
||||
"count": 1,
|
||||
"error": "RuntimeError: boom",
|
||||
}
|
||||
)
|
||||
filer.reconcile()
|
||||
assert journal.unconsumed() != [] # not filed → stays spooled
|
||||
|
||||
|
||||
def test_reconcile_never_raises_when_journal_read_fails():
|
||||
class ExplodingJournal(FakeJournal):
|
||||
def unconsumed(self):
|
||||
raise OSError("disk gone")
|
||||
|
||||
filer = FailureAutoFiler(FakeForge(), ExplodingJournal(), wait=False)
|
||||
filer.reconcile() # must not raise
|
||||
|
||||
|
||||
def test_reconcile_malformed_line_is_skipped_without_crash():
|
||||
# #122 gate: assert against the REAL fake state (the old version ended
|
||||
# in forge_created_not_called(), which returned True unconditionally).
|
||||
journal = FakeJournal()
|
||||
journal.lines.append({"kind": "failure_auto_file_spool", "spool_id": "bad", "tier": 1})
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, journal, wait=False)
|
||||
report = filer.reconcile()
|
||||
assert forge.created == [] # nothing was written to the forge
|
||||
assert forge.commented == []
|
||||
assert forge.searched == []
|
||||
assert report["replayed"] == 1
|
||||
assert report["failed"] == 1
|
||||
assert journal.unconsumed() == [{"kind": "failure_auto_file_spool", "spool_id": "bad", "tier": 1}]
|
||||
@@ -1,290 +0,0 @@
|
||||
"""#122 Task 3: wiring — record_failure filer param, composition root,
|
||||
RUNE_AUTOFILE_OFF, and the frozen call-site compatibility contract.
|
||||
|
||||
The two existing call-site families (capability_broker.py:122,
|
||||
context_broker.py:315) must keep working UNCHANGED when no filer is
|
||||
wired; when the composition root wires one _auto_filer, they file.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from rune import rune_operator
|
||||
from rune import event_bus, subsystem_health
|
||||
from rune.capability_broker import select_for_job
|
||||
from rune.failure_auto_filer import (
|
||||
AUTOFILE_OFF_ENV,
|
||||
FailureAutoFiler,
|
||||
JsonlSpool,
|
||||
)
|
||||
from rune.subsystem_health import SubsystemHealthRegistry, record_failure, set_auto_filer
|
||||
|
||||
from tests.test_failure_auto_filer import FakeForge, FakeJournal
|
||||
|
||||
|
||||
class RecordingFiler:
|
||||
"""Callable filer stand-in matching the callback contract."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls: list[tuple[str, str, int]] = []
|
||||
self.fail_with: Exception | None = None
|
||||
|
||||
def __call__(self, subsystem: str, error, tier: int) -> None:
|
||||
if self.fail_with is not None:
|
||||
raise self.fail_with
|
||||
self.calls.append((subsystem, str(error), int(tier)))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated(tmp_path, monkeypatch):
|
||||
"""Isolate journal + process default filer for every test."""
|
||||
monkeypatch.setattr("rune.event_bus._store_path", lambda: tmp_path / "events.jsonl")
|
||||
monkeypatch.setattr(subsystem_health, "_AUTO_FILER", None)
|
||||
monkeypatch.setattr(rune_operator, "_operator", None)
|
||||
monkeypatch.delenv("FORGE_URL", raising=False)
|
||||
monkeypatch.delenv("FORGE_TOKEN", raising=False)
|
||||
monkeypatch.delenv(AUTOFILE_OFF_ENV, raising=False)
|
||||
yield
|
||||
subsystem_health.set_auto_filer(None)
|
||||
|
||||
|
||||
# ------------------------------------------------- record_failure filer param
|
||||
|
||||
|
||||
def test_filer_param_is_typed_not_bare_any():
|
||||
# #122 gate: the filer callback carries a real protocol type, not Any.
|
||||
import typing
|
||||
|
||||
hints = typing.get_type_hints(subsystem_health.record_failure)
|
||||
assert hints["filer"] == subsystem_health.AutoFiler | None
|
||||
hints_set = typing.get_type_hints(subsystem_health.set_auto_filer)
|
||||
assert hints_set["filer"] == subsystem_health.AutoFiler | None
|
||||
|
||||
|
||||
def test_unwired_record_failure_is_unchanged():
|
||||
# No filer param, no process default: registry entry recorded, return
|
||||
# contract intact, nothing else happens.
|
||||
registry = SubsystemHealthRegistry()
|
||||
assert record_failure("memory_selector", RuntimeError("boom"), tier=1, registry=registry)
|
||||
assert registry.snapshot()["memory_selector"]["failures"] == 1
|
||||
|
||||
|
||||
def test_record_failure_explicit_filer_param_files():
|
||||
filer = RecordingFiler()
|
||||
registry = SubsystemHealthRegistry()
|
||||
record_failure("memory_selector", RuntimeError("boom"), tier=1, registry=registry, filer=filer)
|
||||
assert len(filer.calls) == 1
|
||||
subsystem_name, error, tier = filer.calls[0]
|
||||
assert subsystem_name == "memory_selector" and tier == 1
|
||||
assert "boom" in str(error)
|
||||
assert registry.snapshot()["memory_selector"]["failures"] == 1
|
||||
|
||||
|
||||
def test_record_failure_return_contract_unchanged_with_filer():
|
||||
filer = RecordingFiler()
|
||||
registry = SubsystemHealthRegistry()
|
||||
first = record_failure("memory_selector", RuntimeError("a"), tier=1, registry=registry, filer=filer)
|
||||
second = record_failure("memory_selector", RuntimeError("b"), tier=1, registry=registry, filer=filer)
|
||||
assert first and not second # anti-spam surfacing untouched by wiring
|
||||
assert not record_failure("cron", RuntimeError("x"), tier=2, registry=registry, filer=filer)
|
||||
|
||||
|
||||
def test_filer_exception_never_propagates_from_record_failure():
|
||||
filer = RecordingFiler()
|
||||
filer.fail_with = RuntimeError("filer exploded")
|
||||
registry = SubsystemHealthRegistry()
|
||||
# Must not raise; the registry record still lands.
|
||||
assert record_failure("memory_selector", RuntimeError("boom"), tier=1, registry=registry, filer=filer)
|
||||
assert registry.snapshot()["memory_selector"]["failures"] == 1
|
||||
|
||||
|
||||
def test_process_default_filer_wired_by_composition():
|
||||
filer = RecordingFiler()
|
||||
subsystem_health.set_auto_filer(filer)
|
||||
registry = SubsystemHealthRegistry()
|
||||
record_failure("memory_selector", RuntimeError("boom"), tier=1, registry=registry)
|
||||
assert len(filer.calls) == 1
|
||||
subsystem_name, error, tier = filer.calls[0]
|
||||
assert subsystem_name == "memory_selector" and tier == 1
|
||||
assert "boom" in str(error)
|
||||
|
||||
|
||||
def test_explicit_filer_param_overrides_process_default():
|
||||
default_filer = RecordingFiler()
|
||||
explicit_filer = RecordingFiler()
|
||||
subsystem_health.set_auto_filer(default_filer)
|
||||
registry = SubsystemHealthRegistry()
|
||||
record_failure("memory_selector", RuntimeError("boom"), tier=1, registry=registry, filer=explicit_filer)
|
||||
assert explicit_filer.calls and default_filer.calls == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------- tier gate
|
||||
|
||||
|
||||
def test_t2_reaches_filer_but_filer_gates_it():
|
||||
# The gate lives in ONE place (the filer): record_failure forwards all
|
||||
# tiers; the filer's submit drops T2 before any queue traffic.
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), wait=True)
|
||||
subsystem_health.set_auto_filer(filer)
|
||||
registry = SubsystemHealthRegistry()
|
||||
record_failure("cron", "Connection refused", tier=2, registry=registry)
|
||||
filer.drain(timeout=5)
|
||||
assert forge.searched == []
|
||||
assert forge.created == []
|
||||
assert forge.commented == []
|
||||
|
||||
|
||||
def test_t1_files_through_process_default_filer():
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), wait=True)
|
||||
subsystem_health.set_auto_filer(filer)
|
||||
registry = SubsystemHealthRegistry()
|
||||
record_failure("memory_selector", "RuntimeError: boom", tier=1, registry=registry)
|
||||
filer.drain(timeout=5)
|
||||
assert len(forge.created) == 1
|
||||
assert forge.created[0]["title"].startswith("[auto-failure:memory_selector:RuntimeError]")
|
||||
|
||||
|
||||
# -------------------------------------------------------- call-site contract
|
||||
|
||||
|
||||
def test_capability_broker_call_site_unchanged_when_unwired(monkeypatch):
|
||||
def broken_select(description, need_tools):
|
||||
raise RuntimeError("selector exploded")
|
||||
|
||||
monkeypatch.setattr("rune.capability_broker._select_for_job", broken_select)
|
||||
result = select_for_job("write tests", need_tools=True)
|
||||
assert result["reason"].startswith("fallback full toolset") # degrade-not-crash
|
||||
|
||||
|
||||
def test_capability_broker_call_site_files_when_wired(monkeypatch):
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), wait=True)
|
||||
subsystem_health.set_auto_filer(filer)
|
||||
|
||||
def broken_select(description, need_tools):
|
||||
raise RuntimeError("selector exploded")
|
||||
|
||||
monkeypatch.setattr("rune.capability_broker._select_for_job", broken_select)
|
||||
result = select_for_job("write tests", need_tools=True)
|
||||
assert result["reason"].startswith("fallback full toolset")
|
||||
filer.drain(timeout=5)
|
||||
assert len(forge.created) == 1
|
||||
assert forge.created[0]["title"].startswith("[auto-failure:capability_selection:RuntimeError]")
|
||||
|
||||
|
||||
def test_context_broker_t2_call_site_stays_journal_only_when_wired(monkeypatch):
|
||||
# context_broker.py:315 records history_archive failures at tier 2 —
|
||||
# the frozen kill test through a REAL call site: zero client calls.
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, FakeJournal(), wait=True)
|
||||
subsystem_health.set_auto_filer(filer)
|
||||
registry = SubsystemHealthRegistry()
|
||||
record_failure("history_archive", OSError("disk on fire"), tier=2, registry=registry)
|
||||
filer.drain(timeout=5)
|
||||
assert forge.searched == []
|
||||
assert forge.created == []
|
||||
assert forge.commented == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------- composition root
|
||||
|
||||
|
||||
def _reset_and_get_operator(monkeypatch, forge):
|
||||
monkeypatch.setattr("rune.forge_client.create_forge_client", lambda config=None: forge)
|
||||
return rune_operator.get_operator()
|
||||
|
||||
|
||||
def test_composition_root_wires_one_auto_filer(monkeypatch):
|
||||
forge = FakeForge()
|
||||
monkeypatch.setenv("FORGE_URL", "https://forge.example.com")
|
||||
monkeypatch.setenv("FORGE_TOKEN", "tok")
|
||||
_reset_and_get_operator(monkeypatch, forge)
|
||||
assert isinstance(subsystem_health._AUTO_FILER, FailureAutoFiler)
|
||||
# A wired failure files through the composition-root client.
|
||||
record_failure("memory_selector", "RuntimeError: boom", tier=1)
|
||||
subsystem_health._AUTO_FILER.drain(timeout=5)
|
||||
assert len(forge.created) == 1
|
||||
|
||||
|
||||
def test_composition_root_reconciles_spool_at_startup(monkeypatch, tmp_path):
|
||||
# A spooled line from a previous run lands once the operator boots.
|
||||
forge = FakeForge()
|
||||
monkeypatch.setenv("FORGE_URL", "https://forge.example.com")
|
||||
monkeypatch.setenv("FORGE_TOKEN", "tok")
|
||||
spool = JsonlSpool(tmp_path / "events.jsonl")
|
||||
spool.append(
|
||||
{
|
||||
"kind": "failure_auto_file_spool",
|
||||
"spool_id": "spool-1",
|
||||
"timestamp": "t",
|
||||
"signature": ["memory_selector", "RuntimeError"],
|
||||
"tier": 1,
|
||||
"count": 1,
|
||||
"error": "RuntimeError: boom",
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr("rune.event_bus._store_path", lambda: tmp_path / "events.jsonl")
|
||||
_reset_and_get_operator(monkeypatch, forge)
|
||||
deadline = time.monotonic() + 5
|
||||
while not forge.created and time.monotonic() < deadline:
|
||||
time.sleep(0.02) # startup reconcile runs on its own daemon thread
|
||||
assert len(forge.created) == 1
|
||||
assert spool.unconsumed() == []
|
||||
|
||||
|
||||
def test_composition_root_wiring_never_raises(monkeypatch):
|
||||
def explode(config=None):
|
||||
raise RuntimeError("forge factory broken")
|
||||
|
||||
monkeypatch.setenv("FORGE_URL", "https://forge.example.com")
|
||||
monkeypatch.setenv("FORGE_TOKEN", "tok")
|
||||
monkeypatch.setattr("rune.forge_client.create_forge_client", explode)
|
||||
op = rune_operator.get_operator() # must not raise
|
||||
assert op is not None
|
||||
|
||||
|
||||
def test_rune_autofile_off_disables_entirely(monkeypatch):
|
||||
forge = FakeForge()
|
||||
monkeypatch.setenv("FORGE_URL", "https://forge.example.com")
|
||||
monkeypatch.setenv("FORGE_TOKEN", "tok")
|
||||
monkeypatch.setenv(AUTOFILE_OFF_ENV, "1")
|
||||
_reset_and_get_operator(monkeypatch, forge)
|
||||
assert subsystem_health._AUTO_FILER is None
|
||||
record_failure("memory_selector", "RuntimeError: boom", tier=1)
|
||||
assert forge.searched == []
|
||||
assert forge.created == []
|
||||
|
||||
|
||||
def test_autofile_off_blocks_set_auto_filer(monkeypatch):
|
||||
monkeypatch.setenv(AUTOFILE_OFF_ENV, "1")
|
||||
subsystem_health.set_auto_filer(RecordingFiler())
|
||||
assert subsystem_health._AUTO_FILER is None
|
||||
|
||||
|
||||
def test_unconfigured_forge_wiring_is_noop_by_construction(monkeypatch):
|
||||
# No FORGE_URL/TOKEN: create_forge_client() returns NullAdapter; the
|
||||
# filer is wired but can never touch the network by construction.
|
||||
from rune.forge_client import NullAdapter
|
||||
|
||||
_reset_and_get_operator(monkeypatch, NullAdapter())
|
||||
assert isinstance(subsystem_health._AUTO_FILER, FailureAutoFiler)
|
||||
record_failure("memory_selector", "RuntimeError: boom", tier=1)
|
||||
subsystem_health._AUTO_FILER.drain(timeout=5) # no network, no raise
|
||||
|
||||
|
||||
def test_event_journal_still_receives_canonical_events_after_wiring(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr("rune.event_bus._store_path", lambda: tmp_path / "events.jsonl")
|
||||
forge = FakeForge()
|
||||
filer = FailureAutoFiler(forge, JsonlSpool(tmp_path / "events.jsonl"), wait=True)
|
||||
subsystem_health.set_auto_filer(filer)
|
||||
registry = SubsystemHealthRegistry()
|
||||
record_failure("memory_selector", RuntimeError("boom"), tier=1, registry=registry)
|
||||
filer.drain(timeout=5)
|
||||
# The canonical tier event still lands in the journal.
|
||||
from rune.orchestration.event_store import DurableEventStore
|
||||
|
||||
store = DurableEventStore(tmp_path / "events.jsonl")
|
||||
assert any(r["event_type"] == "capability.failed" for r in store.replay())
|
||||
@@ -26,7 +26,6 @@ def test_null_adapter_is_noop():
|
||||
assert r["created"] is False
|
||||
assert adapter.get_issue_state(1) == {"state": "unknown"}
|
||||
assert adapter.list_open_issues() == []
|
||||
assert adapter.search_issues("query") == []
|
||||
|
||||
|
||||
def test_create_forge_client_unconfigured_returns_null():
|
||||
@@ -66,93 +65,3 @@ def test_gitea_adapter_create_issue_sends_post(monkeypatch):
|
||||
assert "issues" in captured["url"]
|
||||
assert captured["json_data"]["title"] == "Test"
|
||||
assert captured["headers"]["Authorization"] == "token tok"
|
||||
|
||||
|
||||
def test_gitea_adapter_search_issues_uses_repo_scoped_list_endpoint():
|
||||
# #122 gate discharge: Gitea/Forgejo has NO repo-scoped
|
||||
# /issues/search path (verified against live swagger) — the old URL
|
||||
# 404'd on every real forge, so every search raised, everything
|
||||
# spooled, and nothing ever filed. The real request must be the
|
||||
# repo-scoped LIST endpoint, which supports state/type/q server-side
|
||||
# and is paginated.
|
||||
adapter = GiteaForgejoAdapter("https://forge.example.com", "tok", owner="o", repo="r")
|
||||
captured = {}
|
||||
|
||||
class FakeResponse:
|
||||
def json(self):
|
||||
return [{"number": 3, "title": "[auto-failure:cron:Connection]"}]
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
captured["url"] = url
|
||||
return FakeResponse()
|
||||
|
||||
with patch("rune.forge_client.httpx.get", side_effect=fake_get):
|
||||
results = adapter.search_issues("[auto-failure:cron:Connection]")
|
||||
|
||||
assert captured["url"].startswith("https://forge.example.com/api/v1/repos/o/r/issues?")
|
||||
assert "/issues/search" not in captured["url"]
|
||||
assert "state=open" in captured["url"]
|
||||
assert "type=issues" in captured["url"]
|
||||
assert "q=%5Bauto-failure%3Acron%3AConnection%5D" in captured["url"]
|
||||
assert results == [{"number": 3, "title": "[auto-failure:cron:Connection]"}]
|
||||
|
||||
|
||||
def test_gitea_adapter_search_issues_paginates_until_short_page():
|
||||
# The list endpoint is paginated: a 50-item page must be followed by
|
||||
# a page=2 request (a single 50-cap window would silently miss older
|
||||
# failure issues — the same defect list_open_issues has).
|
||||
adapter = GiteaForgejoAdapter("https://forge.example.com", "tok", owner="o", repo="r")
|
||||
pages = {
|
||||
1: [{"number": n, "title": f"issue {n}"} for n in range(50)],
|
||||
2: [{"number": 100, "title": "[auto-failure:cron:Connection] cron"}],
|
||||
}
|
||||
urls = []
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def fake_get(url, **kwargs):
|
||||
urls.append(url)
|
||||
return FakeResponse(pages[1 if "page=1" in url else 2])
|
||||
|
||||
with patch("rune.forge_client.httpx.get", side_effect=fake_get):
|
||||
results = adapter.search_issues("[auto-failure:cron:Connection]")
|
||||
|
||||
assert len(urls) == 2
|
||||
assert any("page=2" in u for u in urls)
|
||||
assert results == [{"number": 100, "title": "[auto-failure:cron:Connection] cron"}]
|
||||
|
||||
|
||||
def test_gitea_adapter_search_filters_to_marker_hits_client_side():
|
||||
# Server q= matches the title (and fuzzy keywords); the client-side
|
||||
# contains-check keeps title/body marker hits and drops fuzzy noise
|
||||
# so a quoted marker in an unrelated issue's body never dedups falsely.
|
||||
adapter = GiteaForgejoAdapter("https://forge.example.com", "tok", owner="o", repo="r")
|
||||
marker = "[auto-failure:cron:Connection]"
|
||||
payload = [
|
||||
{"number": 1, "title": f"{marker} cron", "body": "x"},
|
||||
{"number": 2, "title": "plain title", "body": f"quoted {marker} in body"},
|
||||
{"number": 3, "title": "mention of connection only", "body": "unrelated"},
|
||||
]
|
||||
|
||||
class FakeResponse:
|
||||
def json(self):
|
||||
return payload
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
with patch("rune.forge_client.httpx.get", return_value=FakeResponse()):
|
||||
results = adapter.search_issues(marker)
|
||||
|
||||
assert [r["number"] for r in results] == [1, 2]
|
||||
@@ -1,155 +0,0 @@
|
||||
"""Unit tests for scripts/merge_robot.py — the fail-loud merge-on-green logic.
|
||||
|
||||
Bug-fix contract red-first: the original inline robot swallowed merge refusals
|
||||
(print-and-continue, exit 0) — the silent no-op defect. These tests freeze the
|
||||
fail-loud behavior: refusals fail the job, identity is verified first, every
|
||||
scan produces a decision line.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SPEC = importlib.util.spec_from_file_location("merge_robot", ROOT / "scripts" / "merge_robot.py")
|
||||
mr = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(mr)
|
||||
|
||||
|
||||
class FakeForge:
|
||||
"""Simulates the SERVER contract (not the client's logic — no circularity)."""
|
||||
|
||||
def __init__(self, prs, statuses, merge_results):
|
||||
self.prs = prs
|
||||
self.statuses = statuses # sha -> state
|
||||
self.merge_results = merge_results # pr_number -> (status, body)
|
||||
self.mergeable: dict[int, bool] = {} # pr_number -> server-reported mergeable
|
||||
self.calls: list[tuple[str, str]] = []
|
||||
|
||||
def get(self, path):
|
||||
self.calls.append(("GET", path))
|
||||
if path == "/user":
|
||||
return 200, {"login": "gitea-actions-robot"}
|
||||
if path.startswith("/pulls?state=open"):
|
||||
return 200, self.prs
|
||||
if path.startswith("/commits/"):
|
||||
sha = path.split("/")[2]
|
||||
return 200, {"state": self.statuses.get(sha, "pending")}
|
||||
if path.startswith("/pulls/") and not path.endswith("/merge"):
|
||||
n = int(path.split("/")[2])
|
||||
for pr in self.prs:
|
||||
if pr["number"] == n:
|
||||
# server truth: GET /pulls/{n} includes mergeable (default True)
|
||||
return 200, {**pr, "mergeable": self.mergeable.get(n, True)}
|
||||
return 404, {}
|
||||
raise AssertionError(f"unexpected GET {path}")
|
||||
|
||||
def post(self, path, data):
|
||||
self.calls.append(("POST", path))
|
||||
if path.endswith("/merge"):
|
||||
n = int(path.split("/")[2])
|
||||
return self.merge_results.get(n, (200, {}))
|
||||
raise AssertionError(f"unexpected POST {path}")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def forge(monkeypatch):
|
||||
f = FakeForge([], {}, {})
|
||||
|
||||
def fake_api(base, token, path, method="GET", data=None, ok=(200,)):
|
||||
f.calls.append((method, path))
|
||||
if method == "GET":
|
||||
status, body = f.get(path)
|
||||
else:
|
||||
status, body = f.post(path, data)
|
||||
return status, body
|
||||
|
||||
monkeypatch.setattr(mr, "_api", fake_api)
|
||||
return f
|
||||
|
||||
|
||||
def _pr(n, sha, state="success", mergeable=True):
|
||||
return {"number": n, "head": {"sha": sha}}, {sha: state}, mergeable
|
||||
|
||||
|
||||
def test_identity_verified_before_any_merge(forge, capsys):
|
||||
forge.prs = [{"number": 1, "head": {"sha": "a"}}]
|
||||
forge.statuses = {"a": "pending"}
|
||||
mr.main_with(["merge_robot.py", "tok", "http://x/api/v1/repos/o/r"])
|
||||
assert ("GET", "/user") == forge.calls[0], "identity check must be the first call"
|
||||
|
||||
|
||||
def test_merge_refusal_403_fails_job(forge, capsys):
|
||||
forge.prs = [{"number": 7, "head": {"sha": "s7"}}]
|
||||
forge.statuses = {"s7": "success"}
|
||||
forge.merge_results = {7: (403, {"message": "token does not have required scope"})}
|
||||
with pytest.raises(mr.MergeRefused, match="403"):
|
||||
mr.decide("base", "tok", forge.prs[0])
|
||||
|
||||
|
||||
def test_merge_refusal_405_unexpected_fails_job(forge):
|
||||
forge.prs = [{"number": 8, "head": {"sha": "s8"}}]
|
||||
forge.statuses = {"s8": "success"}
|
||||
forge.merge_results = {8: (405, {"message": "protected branch rejects"})}
|
||||
with pytest.raises(mr.MergeRefused, match="405"):
|
||||
mr.decide("base", "tok", forge.prs[0])
|
||||
|
||||
|
||||
def test_merge_success_returns_merged(forge):
|
||||
forge.prs = [{"number": 9, "head": {"sha": "s9"}}]
|
||||
forge.statuses = {"s9": "success"}
|
||||
forge.merge_results = {9: (200, {})}
|
||||
assert "MERGED" in mr.decide("base", "tok", forge.prs[0])
|
||||
|
||||
|
||||
def test_pending_pr_skipped_not_merged(forge):
|
||||
forge.prs = [{"number": 10, "head": {"sha": "s10"}}]
|
||||
forge.statuses = {"s10": "pending"}
|
||||
assert "skipped" in mr.decide("base", "tok", forge.prs[0])
|
||||
assert not any(m == "POST" for m, _ in forge.calls), "must not attempt merge on pending"
|
||||
|
||||
|
||||
def test_conflict_pr_skipped(forge):
|
||||
forge.prs = [{"number": 11, "head": {"sha": "s11"}}]
|
||||
forge.statuses = {"s11": "success"}
|
||||
forge.mergeable = {11: False} # server reports conflict
|
||||
assert "skipped" in mr.decide("base", "tok", forge.prs[0])
|
||||
assert not any(m == "POST" for m, _ in forge.calls)
|
||||
|
||||
|
||||
def test_behind_base_405_is_skip_not_crash(forge):
|
||||
"""405 'behind the base branch' = needs branch update, not a robot failure."""
|
||||
forge.prs = [{"number": 12, "head": {"sha": "s12"}}]
|
||||
forge.statuses = {"s12": "success"}
|
||||
forge.merge_results = {12: (405, {"message": "The head branch is behind the base branch"})}
|
||||
assert "skipped" in mr.decide("base", "tok", forge.prs[0])
|
||||
|
||||
|
||||
def test_main_exits_1_on_refusal(forge, monkeypatch):
|
||||
forge.prs = [{"number": 13, "head": {"sha": "s13"}}]
|
||||
forge.statuses = {"s13": "success"}
|
||||
forge.merge_results = {13: (403, {"message": "no scope"})}
|
||||
|
||||
def fake_scan(base, token):
|
||||
return forge.prs
|
||||
|
||||
monkeypatch.setattr(mr, "scan", fake_scan)
|
||||
rc = mr.main_with(["merge_robot.py", "tok", "base"])
|
||||
assert rc == 1, "refusal must fail the job (exit 1) — the silent no-op defect"
|
||||
|
||||
|
||||
def test_main_exits_0_all_merged(forge, monkeypatch):
|
||||
forge.prs = [{"number": 14, "head": {"sha": "s14"}}]
|
||||
forge.statuses = {"s14": "success"}
|
||||
forge.merge_results = {14: (200, {})}
|
||||
|
||||
def fake_scan(base, token):
|
||||
return forge.prs
|
||||
|
||||
monkeypatch.setattr(mr, "scan", fake_scan)
|
||||
assert mr.main_with(["merge_robot.py", "tok", "base"]) == 0
|
||||
@@ -20,12 +20,7 @@ def _ollama_up() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _ollama_up() or os.environ.get("GITHUB_ACTIONS") == "true",
|
||||
reason=("Ollama not reachable, or CI: the live-model E2E is nondeterministic "
|
||||
"when a runner host has Ollama (shell-syntax approval gate); CI skips it "
|
||||
"exactly as the Pi runner did when Ollama was unreachable"),
|
||||
)
|
||||
pytestmark = pytest.mark.skipif(not _ollama_up(), reason="Ollama not reachable")
|
||||
|
||||
|
||||
def test_qwen3_14b_fixes_broken_add_end_to_end(tmp_path):
|
||||
|
||||
@@ -1,889 +0,0 @@
|
||||
"""P1-01 (#24) frozen tests — CapabilityBroker becomes the selection authority.
|
||||
|
||||
Frozen tests are the spec. Each task's tests land red first, then the
|
||||
implementation makes them green. Covers:
|
||||
|
||||
- Task 1: per-tool metadata (risk/preconditions/side_effects) on TOOLS and
|
||||
capability-card synthesis + rebuild-on-empty/stale validation;
|
||||
- Task 2: ``select_for_job`` with the verified floor;
|
||||
- Task 3: operator/runtime exposure wiring, ``find_capability`` escape
|
||||
hatch, and trace schema v3 fields;
|
||||
- Task 4: composition registration of the ``capability_selection`` authority.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
from rune import capability_broker as cb # noqa: E402
|
||||
from rune.tool_executor import TOOLS # noqa: E402
|
||||
|
||||
|
||||
VALID_RISKS = {"read-only", "write", "irreversible"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 1 — per-tool metadata + card synthesis/validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_every_tool_entry_carries_risk_preconditions_side_effects():
|
||||
assert TOOLS, "tool registry must not be empty"
|
||||
for tool in TOOLS:
|
||||
assert tool.get("risk") in VALID_RISKS, f"{tool['name']}: bad risk"
|
||||
assert isinstance(tool.get("preconditions"), list), f"{tool['name']}: preconditions"
|
||||
assert isinstance(tool.get("side_effects"), list), f"{tool['name']}: side_effects"
|
||||
|
||||
|
||||
def test_native_cards_are_synthesized_when_cards_file_empty(tmp_path, monkeypatch):
|
||||
cards_file = tmp_path / "capability_cards.json"
|
||||
cards_file.write_text("[]")
|
||||
monkeypatch.setattr(cb, "CARDS_FILE", str(cards_file))
|
||||
monkeypatch.setattr(cb, "REGISTRY", str(tmp_path / "no-registry.json"))
|
||||
|
||||
loaded = cb.load_cards()
|
||||
names = {card["name"] for card in loaded}
|
||||
for tool in TOOLS:
|
||||
assert tool["name"] in names, f"native card missing for {tool['name']}"
|
||||
assert any(card.get("native_tool") for card in loaded), "native cards must be flagged"
|
||||
|
||||
|
||||
def test_native_cards_carry_tool_metadata(tmp_path, monkeypatch):
|
||||
cards_file = tmp_path / "capability_cards.json"
|
||||
cards_file.write_text("[]")
|
||||
monkeypatch.setattr(cb, "CARDS_FILE", str(cards_file))
|
||||
monkeypatch.setattr(cb, "REGISTRY", str(tmp_path / "no-registry.json"))
|
||||
|
||||
by_name = {c["name"]: c for c in cb.load_cards() if c.get("native_tool")}
|
||||
for tool in TOOLS:
|
||||
card = by_name[tool["name"]]
|
||||
assert card["description"] == tool["description"]
|
||||
assert card["risk"] == tool["risk"]
|
||||
assert card["preconditions"] == tool["preconditions"]
|
||||
assert card["side_effects"] == tool["side_effects"]
|
||||
|
||||
|
||||
def test_load_cards_rebuilds_when_file_missing(tmp_path, monkeypatch):
|
||||
cards_file = tmp_path / "missing.json"
|
||||
monkeypatch.setattr(cb, "CARDS_FILE", str(cards_file))
|
||||
monkeypatch.setattr(cb, "REGISTRY", str(tmp_path / "no-registry.json"))
|
||||
|
||||
loaded = cb.load_cards()
|
||||
assert any(card.get("native_tool") for card in loaded)
|
||||
assert cards_file.exists(), "rebuild must persist the synthesized cards"
|
||||
|
||||
|
||||
def test_load_cards_rebuilds_when_stale_vs_registry(tmp_path, monkeypatch):
|
||||
cards_file = tmp_path / "capability_cards.json"
|
||||
cards_file.write_text(json.dumps([{"name": "old_plugin_card", "description": "stale"}]))
|
||||
registry = tmp_path / ".microkernel.json"
|
||||
registry.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"plugins": {
|
||||
"memory_recall": {
|
||||
"manifest": {
|
||||
"description": "recall stored memory",
|
||||
"capabilities": ["memory_recall"],
|
||||
"hooks": {},
|
||||
"_file": "memory_recall.py",
|
||||
"priority": 10,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
old = time.time() - 1000
|
||||
os.utime(cards_file, (old, old))
|
||||
monkeypatch.setattr(cb, "CARDS_FILE", str(cards_file))
|
||||
monkeypatch.setattr(cb, "REGISTRY", str(registry))
|
||||
|
||||
loaded = cb.load_cards()
|
||||
names = {card["name"] for card in loaded}
|
||||
assert "old_plugin_card" not in names, "stale cards must be rebuilt away"
|
||||
assert "memory_recall" in names, "registry plugins must be rebuilt in"
|
||||
assert any(card.get("native_tool") for card in loaded)
|
||||
|
||||
|
||||
def test_load_cards_keeps_fresh_nonempty_file(tmp_path, monkeypatch):
|
||||
fresh = [
|
||||
{
|
||||
"name": "read_file",
|
||||
"description": "d",
|
||||
"capabilities": ["read_file"],
|
||||
"priority": 100,
|
||||
"risk": "read-only",
|
||||
"preconditions": [],
|
||||
"side_effects": [],
|
||||
"native_tool": True,
|
||||
}
|
||||
]
|
||||
cards_file = tmp_path / "capability_cards.json"
|
||||
cards_file.write_text(json.dumps(fresh))
|
||||
monkeypatch.setattr(cb, "CARDS_FILE", str(cards_file))
|
||||
monkeypatch.setattr(cb, "REGISTRY", str(tmp_path / "no-registry.json"))
|
||||
|
||||
assert cb.load_cards() == fresh, "fresh non-empty cards must not be rebuilt"
|
||||
|
||||
|
||||
def test_load_cards_rebuilds_on_corrupt_json(tmp_path, monkeypatch):
|
||||
cards_file = tmp_path / "capability_cards.json"
|
||||
cards_file.write_text("{not json")
|
||||
monkeypatch.setattr(cb, "CARDS_FILE", str(cards_file))
|
||||
monkeypatch.setattr(cb, "REGISTRY", str(tmp_path / "no-registry.json"))
|
||||
|
||||
loaded = cb.load_cards()
|
||||
assert any(card.get("native_tool") for card in loaded)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 2 — select_for_job + verified floor
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FLOOR = cb.FLOOR_TOOLS
|
||||
TASK_TOOLS = {"read_file", "write_file", "edit_file", "list_directory", "grep_files"}
|
||||
|
||||
|
||||
def test_floor_constant_names_exact_required_tools():
|
||||
assert set(FLOOR) == TASK_TOOLS, "floor must be exactly the five T1 file tools"
|
||||
assert cb.ESCAPE_HATCH_TOOL == "find_capability"
|
||||
|
||||
|
||||
def test_select_for_job_coding_task_returns_floor_plus_matches():
|
||||
result = cb.select_for_job("fix the bug in parse_user.py and add a test")
|
||||
names = [t["name"] for t in result["tools"]]
|
||||
assert TASK_TOOLS <= set(names), "floor must always be present"
|
||||
assert "find_capability" in names, "escape hatch must always be present"
|
||||
assert 5 <= len(names) <= 10, f"selection size {len(names)} outside 5..10"
|
||||
assert names, "selection must never be empty"
|
||||
|
||||
|
||||
def test_select_for_job_zero_score_falls_back_to_floor():
|
||||
gibberish = "zzzqqq xxxyyy wwww vvvv"
|
||||
result = cb.select_for_job(gibberish)
|
||||
names = [t["name"] for t in result["tools"]]
|
||||
assert set(names) == TASK_TOOLS | {"find_capability"}
|
||||
assert result["reason"] == "floor (no matches)"
|
||||
|
||||
|
||||
def test_select_for_job_reason_strings_recorded():
|
||||
result = cb.select_for_job("fix the failing test in module.py")
|
||||
assert isinstance(result["reason"], str) and result["reason"]
|
||||
assert result["reason"].startswith("floor"), "reason must record the floor basis"
|
||||
|
||||
|
||||
def test_select_for_job_find_capability_always_present():
|
||||
for description in ("fix code", "remember this", "research topic", ""):
|
||||
result = cb.select_for_job(description)
|
||||
names = [t["name"] for t in result["tools"]]
|
||||
assert "find_capability" in names
|
||||
|
||||
|
||||
def test_select_for_job_is_deterministic():
|
||||
a = cb.select_for_job("fix the bug in parse.py")
|
||||
b = cb.select_for_job("fix the bug in parse.py")
|
||||
assert [t["name"] for t in a["tools"]] == [t["name"] for t in b["tools"]]
|
||||
assert a["reason"] == b["reason"]
|
||||
|
||||
|
||||
def test_select_for_job_selection_derived_from_full_tools_payload():
|
||||
"""Selected entries are the actual schema dicts (ride the tools param)."""
|
||||
result = cb.select_for_job("fix the bug in parse.py")
|
||||
by_name = {t["name"]: t for t in TOOLS}
|
||||
for selected in result["tools"]:
|
||||
assert selected is by_name[selected["name"]], f"{selected['name']} must be the real schema object"
|
||||
|
||||
|
||||
def test_select_for_job_floor_survives_any_query_and_set_stays_bounded():
|
||||
"""Floor is policy, not score: present for every query, set ≤ 10."""
|
||||
for description in (
|
||||
"grep for the word banana across files",
|
||||
"remember this decision for later",
|
||||
"research the history of this topic thoroughly",
|
||||
"",
|
||||
):
|
||||
result = cb.select_for_job(description)
|
||||
names = [t["name"] for t in result["tools"]]
|
||||
assert TASK_TOOLS <= set(names)
|
||||
assert "find_capability" in names
|
||||
assert len(names) <= 10
|
||||
|
||||
|
||||
def test_select_for_job_records_task_family():
|
||||
result = cb.select_for_job("fix the bug in parse.py")
|
||||
assert result["task_family"] == "code"
|
||||
|
||||
|
||||
def test_select_for_job_degrades_to_full_tools_on_broker_failure(monkeypatch):
|
||||
def boom():
|
||||
raise RuntimeError("broker exploded")
|
||||
|
||||
monkeypatch.setattr(cb, "_scored_tools", boom)
|
||||
result = cb.select_for_job("fix the bug in parse.py")
|
||||
assert [t["name"] for t in result["tools"]] == [t["name"] for t in TOOLS]
|
||||
assert result["reason"].startswith("fallback full toolset")
|
||||
|
||||
|
||||
def test_select_for_job_plugin_cards_do_not_invent_tool_schemas(monkeypatch):
|
||||
"""Plugin capabilities ride the escape hatch, not the tools param:
|
||||
the selection contains only real TOOLS schemas, floor intact."""
|
||||
fake_cards = [
|
||||
{
|
||||
"name": "memory_recall",
|
||||
"description": "recall stored memory about past sessions",
|
||||
"capabilities": ["memory_recall"],
|
||||
"priority": 5,
|
||||
"semantic_examples": ["What do I remember about X?"],
|
||||
"risk": "read-only",
|
||||
"searchable_text": "memory recall recall stored memory remember",
|
||||
}
|
||||
]
|
||||
monkeypatch.setattr(cb, "load_cards", lambda: fake_cards)
|
||||
result = cb.select_for_job("what do I remember about this session")
|
||||
names = [t["name"] for t in result["tools"]]
|
||||
by_name = {t["name"] for t in TOOLS}
|
||||
assert set(names) <= by_name, "selection must never invent a non-schema tool"
|
||||
assert set(TASK_TOOLS) <= set(names), "floor always present"
|
||||
assert "find_capability" in names, "escape hatch always present"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 3 — operator/runtime wiring + find_capability escape hatch + trace v3
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
import copy # noqa: E402
|
||||
import unittest.mock as mock # noqa: E402
|
||||
|
||||
from rune import local_tool_runtime as ltr # noqa: E402
|
||||
from rune import rune_operator # noqa: E402
|
||||
from rune.local_tool_runtime import LocalToolRuntime # noqa: E402
|
||||
from rune.tool_protocol import to_ollama_tools # noqa: E402
|
||||
|
||||
|
||||
class _FakeTransport:
|
||||
def __init__(self, responses):
|
||||
self.responses = list(responses)
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def supports_tools(self, model: str) -> bool:
|
||||
return True
|
||||
|
||||
def supports_thinking(self, model: str) -> bool:
|
||||
return True
|
||||
|
||||
def chat(self, *, model, messages, tools, options=None, think=None):
|
||||
self.calls.append({"tools": tools, "messages": messages})
|
||||
response = self.responses.pop(0)
|
||||
return copy.deepcopy(response)
|
||||
|
||||
|
||||
def _native_call(call_id, name, arguments):
|
||||
return {"id": call_id, "function": {"name": name, "arguments": arguments}}
|
||||
|
||||
|
||||
def _record_tool(name, arguments, call_id="c1"):
|
||||
"""Assistant message carrying one native tool call."""
|
||||
return {
|
||||
"content": "",
|
||||
"tool_calls": [_native_call(call_id, name, arguments)],
|
||||
}
|
||||
|
||||
|
||||
def test_runtime_exposes_only_selected_schemas():
|
||||
"""Selection flows to the tools param: payload shrinks below 19."""
|
||||
selected = cb.select_for_job("fix the bug in parse.py")
|
||||
assert len(selected["tools"]) < len(TOOLS), "selection must shrink the set"
|
||||
runtime = LocalToolRuntime(
|
||||
transport=_FakeTransport([]),
|
||||
executor=lambda n, a: {"success": True},
|
||||
model="m",
|
||||
tools=selected["tools"],
|
||||
)
|
||||
payload = to_ollama_tools(runtime.tools)
|
||||
names = {item["function"]["name"] for item in payload}
|
||||
assert names == set(selected["names"])
|
||||
assert len(payload) < len(to_ollama_tools(TOOLS))
|
||||
|
||||
|
||||
def test_default_runtime_still_exposes_full_tools():
|
||||
"""No selection passed → full TOOLS (today's behavior, degrade path)."""
|
||||
runtime = LocalToolRuntime(
|
||||
transport=_FakeTransport([]),
|
||||
executor=lambda n, a: {"success": True},
|
||||
model="m",
|
||||
)
|
||||
assert len(runtime.tools) == len(TOOLS)
|
||||
|
||||
|
||||
def test_inference_trace_schema_v3_fields():
|
||||
from rune import inference_trace as it
|
||||
|
||||
# P1-02 (#25) bumped the schema to v4 (additive); assert against the
|
||||
# module constant, not a pinned literal.
|
||||
assert it.INFERENCE_TRACE_SCHEMA_VERSION == 4
|
||||
trace = it.InferenceTrace(
|
||||
schema_version=it.INFERENCE_TRACE_SCHEMA_VERSION,
|
||||
provider="ollama",
|
||||
model="m",
|
||||
task_id="t",
|
||||
channel="c",
|
||||
resolved_context_window=0,
|
||||
input_budget=0,
|
||||
model_calls=0,
|
||||
prompt_tokens_estimated=0,
|
||||
prompt_tokens_served=0,
|
||||
output_tokens=0,
|
||||
latency_ms=0.0,
|
||||
model_time_ms=0.0,
|
||||
per_block_tokens={},
|
||||
dropped_blocks=[],
|
||||
tools_exposed=[],
|
||||
memories_exposed=[],
|
||||
memory_tokens=0,
|
||||
degraded=[],
|
||||
cache=None,
|
||||
cost_usd=None,
|
||||
cost_basis="local-unpriced",
|
||||
success=True,
|
||||
error="",
|
||||
)
|
||||
assert trace.selected_tools == []
|
||||
assert trace.capability_reason == ""
|
||||
assert trace.find_capability_calls == 0
|
||||
|
||||
|
||||
def test_operator_trace_record_carries_capability_fields(tmp_path, monkeypatch):
|
||||
"""A scripted operator run persists v3 capability fields."""
|
||||
monkeypatch.setenv("RUNE_HOME", str(tmp_path / "home"))
|
||||
(tmp_path / "home" / "state").mkdir(parents=True, exist_ok=True)
|
||||
from rune import inference_trace as it
|
||||
|
||||
storage = tmp_path / "operator.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
trace = {
|
||||
"model_id": "m",
|
||||
"window": 40960,
|
||||
"budget": 32768,
|
||||
"blocks": [
|
||||
{"name": "persona", "tokens": 10, "admitted": True, "reason": ""},
|
||||
{"name": "user", "tokens": 5, "admitted": True, "reason": ""},
|
||||
],
|
||||
"admitted_total": 15,
|
||||
"dropped": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"compaction": [],
|
||||
"elapsed_ms": 1.0,
|
||||
}
|
||||
capsule = mock.Mock(
|
||||
token_count=15,
|
||||
active_objective="",
|
||||
blocks=[],
|
||||
to_messages=lambda: [{"role": "user", "content": "do it"}],
|
||||
)
|
||||
broker = mock.Mock(assemble=lambda task, session, model_id: (capsule, trace))
|
||||
response = rune_operator.OperatorResponse(success=True, user_text="done", model="m")
|
||||
|
||||
class _RT:
|
||||
transport = mock.Mock(provider="ollama")
|
||||
tool_names = frozenset({"read_file", "find_capability"})
|
||||
|
||||
def run(self, messages, **kwargs):
|
||||
return response
|
||||
|
||||
rt = _RT()
|
||||
with (
|
||||
mock.patch.object(op, "_context_broker", return_value=broker),
|
||||
mock.patch.object(rune_operator.model_profiles, "get_or_create_profile", return_value=mock.Mock(safety_tier=0)),
|
||||
mock.patch.object(rune_operator.model_profiles, "record_interaction"),
|
||||
mock.patch.object(rune_operator, "LocalToolRuntime", return_value=rt),
|
||||
mock.patch.object(rune_operator, "OllamaChatTransport", return_value=rt.transport),
|
||||
mock.patch("rune.local_tool_runtime.context_length", lambda model_: 40960),
|
||||
mock.patch.object(
|
||||
rune_operator.safety_override.SafetyOverride,
|
||||
"get_params",
|
||||
lambda self, text: {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""},
|
||||
),
|
||||
mock.patch.object(rune_operator.safety_override.SafetyOverride, "is_refusal", lambda self, text: False),
|
||||
mock.patch.object(op, "_log_token_usage"),
|
||||
mock.patch("rune.event_bus._store_path", lambda: tmp_path / "journal-events.jsonl"),
|
||||
):
|
||||
envelope = op.send_task("fix this code path")
|
||||
|
||||
assert envelope.get("success") is True
|
||||
records = it.read_traces(trace_dir=tmp_path / "home" / "state")
|
||||
assert records, "durable trace missing"
|
||||
record = records[-1]
|
||||
assert record["schema_version"] == 4
|
||||
assert record["selected_tools"] == cb.select_for_job("fix this code path")["names"]
|
||||
assert "capability_reason" in record and record["capability_reason"]
|
||||
assert record["find_capability_calls"] == 0
|
||||
|
||||
|
||||
def test_operator_trace_record_carries_selected_tools(tmp_path, monkeypatch):
|
||||
"""The scripted run's LocalToolRuntime receives the selected set AND
|
||||
the trace carries the v3 capability fields."""
|
||||
monkeypatch.setenv("RUNE_HOME", str(tmp_path / "home2"))
|
||||
(tmp_path / "home2" / "state").mkdir(parents=True, exist_ok=True)
|
||||
from rune import inference_trace as it
|
||||
|
||||
storage = tmp_path / "operator2.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
trace = {
|
||||
"model_id": "m",
|
||||
"window": 40960,
|
||||
"budget": 32768,
|
||||
"blocks": [
|
||||
{"name": "persona", "tokens": 10, "admitted": True, "reason": ""},
|
||||
{"name": "user", "tokens": 5, "admitted": True, "reason": ""},
|
||||
],
|
||||
"admitted_total": 15,
|
||||
"dropped": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"compaction": [],
|
||||
"elapsed_ms": 1.0,
|
||||
}
|
||||
capsule = mock.Mock(
|
||||
token_count=15,
|
||||
active_objective="",
|
||||
blocks=[],
|
||||
to_messages=lambda: [{"role": "user", "content": "do it"}],
|
||||
)
|
||||
broker = mock.Mock(assemble=lambda task, session, model_id: (capsule, trace))
|
||||
response = rune_operator.OperatorResponse(success=True, user_text="done", model="m")
|
||||
captured: dict = {}
|
||||
|
||||
class _RT:
|
||||
transport = mock.Mock(provider="ollama")
|
||||
tool_names = frozenset()
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
|
||||
def run(self, messages, **kwargs):
|
||||
return response
|
||||
|
||||
with (
|
||||
mock.patch.object(op, "_context_broker", return_value=broker),
|
||||
mock.patch.object(rune_operator.model_profiles, "get_or_create_profile", return_value=mock.Mock(safety_tier=0)),
|
||||
mock.patch.object(rune_operator.model_profiles, "record_interaction"),
|
||||
mock.patch.object(rune_operator, "LocalToolRuntime", _RT),
|
||||
mock.patch.object(rune_operator, "OllamaChatTransport", return_value=mock.Mock(provider="ollama")),
|
||||
mock.patch("rune.local_tool_runtime.context_length", lambda model_: 40960),
|
||||
mock.patch.object(
|
||||
rune_operator.safety_override.SafetyOverride,
|
||||
"get_params",
|
||||
lambda self, text: {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""},
|
||||
),
|
||||
mock.patch.object(rune_operator.safety_override.SafetyOverride, "is_refusal", lambda self, text: False),
|
||||
mock.patch.object(op, "_log_token_usage"),
|
||||
mock.patch("rune.event_bus._store_path", lambda: tmp_path / "journal2-events.jsonl"),
|
||||
):
|
||||
envelope = op.send_task("fix this code path")
|
||||
|
||||
assert envelope.get("success") is True
|
||||
tool_names = [t["name"] for t in captured.get("tools", [])]
|
||||
assert tool_names == cb.select_for_job("fix this code path")["names"], (
|
||||
"runtime construction site must receive the selected set"
|
||||
)
|
||||
records = it.read_traces(trace_dir=tmp_path / "home2" / "state")
|
||||
assert records
|
||||
record = records[-1]
|
||||
assert record["schema_version"] == 4
|
||||
assert record["selected_tools"] == tool_names
|
||||
assert record["capability_reason"]
|
||||
assert record["find_capability_calls"] == 0
|
||||
|
||||
|
||||
def test_runtime_for_model_reuses_pending_exposed_set(tmp_path):
|
||||
"""Resume round-trip: the persisted exposed set (selected + any tools
|
||||
dynamically exposed via find_capability) is rebuilt exactly."""
|
||||
storage = tmp_path / "operator3.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
op.state.pending_runtime_state = {"exposed_tools": ["read_file", "find_capability"]}
|
||||
with mock.patch.object(rune_operator, "LocalToolRuntime", return_value=mock.Mock()) as rt_type:
|
||||
op._runtime_for_model("m1")
|
||||
_, kwargs = rt_type.call_args
|
||||
assert [t["name"] for t in kwargs["tools"]] == ["read_file", "find_capability"]
|
||||
|
||||
|
||||
def test_runtime_for_model_reselects_from_description_without_pending(tmp_path):
|
||||
storage = tmp_path / "operator4.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
with mock.patch.object(rune_operator, "LocalToolRuntime", return_value=mock.Mock()) as rt_type:
|
||||
op._runtime_for_model("m1", "fix the bug in parse.py")
|
||||
_, kwargs = rt_type.call_args
|
||||
assert [t["name"] for t in kwargs["tools"]] == cb.select_for_job("fix the bug in parse.py")["names"]
|
||||
|
||||
|
||||
def test_contributor_tools_reflects_selection(tmp_path):
|
||||
storage = tmp_path / "operator5.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
task = mock.Mock(description="fix the bug in parse.py")
|
||||
block = op._contributor_tools(task, {})
|
||||
assert "find_capability" in block.content, "escape hatch must be advertised"
|
||||
for name in ("lint_file", "git_commit", "repo_map", "run_command", "read_persona"):
|
||||
assert name not in block.content, f"unselected {name} must cost zero prompt tokens"
|
||||
for name in cb.select_for_job("fix the bug in parse.py")["names"]:
|
||||
assert name in block.content
|
||||
|
||||
|
||||
def test_contributor_tools_names_track_selection_for_tool_heavy_task(tmp_path):
|
||||
storage = tmp_path / "operator6.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
task = mock.Mock(description="run the test suite and commit")
|
||||
block = op._contributor_tools(task, {})
|
||||
selection_names = cb.select_for_job("run the test suite and commit")["names"]
|
||||
for name in selection_names:
|
||||
assert name in block.content
|
||||
for name in ("repo_map", "read_persona", "diagnose_behavior"):
|
||||
assert name not in block.content
|
||||
|
||||
|
||||
def test_unknown_tool_call_returns_guidance_error():
|
||||
"""Unknown-tool failure must name find_capability (one-step recovery)."""
|
||||
transport = _FakeTransport([_record_tool("nonexistent_tool", {"x": 1}, call_id="c1")])
|
||||
runtime = LocalToolRuntime(
|
||||
transport=transport,
|
||||
executor=lambda n, a: {"success": True},
|
||||
model="m",
|
||||
tools=cb.select_for_job("fix code")["tools"],
|
||||
)
|
||||
result = runtime.run([{"role": "user", "content": "go"}])
|
||||
assert result.success is False
|
||||
assert "find_capability" in (result.error or "")
|
||||
|
||||
|
||||
def test_find_capability_execution_and_same_run_discovery():
|
||||
"""find_capability returns matching schemas; the SAME run then calls
|
||||
the discovered tool (dynamic exposure within the iteration budget)."""
|
||||
executor = ltr2_exec = ltr._find_capability_executor if hasattr(ltr, "FIND_CAPABILITY_EXECUTOR") else None
|
||||
responses = [
|
||||
_record_tool("find_capability", {"query": "map the repository structure"}, call_id="c1"),
|
||||
_record_tool("repo_map", {"path": "."}, call_id="c2"),
|
||||
{"content": "tests ran"},
|
||||
]
|
||||
transport = _FakeTransport(responses)
|
||||
runtime = LocalToolRuntime(
|
||||
transport=transport,
|
||||
executor=executor,
|
||||
model="m",
|
||||
tools=cb.select_for_job("fix code")["tools"],
|
||||
)
|
||||
result = runtime.run([{"role": "user", "content": "run the tests"}])
|
||||
assert result.success is True, result.error
|
||||
assert result.user_text == "tests ran"
|
||||
# The second chat call must have carried the discovered tool schema.
|
||||
second_tools = transport.calls[1]["tools"]
|
||||
second_names = {item["function"]["name"] for item in second_tools}
|
||||
assert "repo_map" in second_names, "discovered tool must be exposed in-run"
|
||||
|
||||
|
||||
def test_find_capability_result_carries_schema_summary():
|
||||
executor = ltr._find_capability_executor
|
||||
result = executor("find_capability", {"query": "commit my changes"})
|
||||
assert result.get("success") is True
|
||||
text = result.get("result", "")
|
||||
assert "git_commit" in text
|
||||
assert "message" in text, "argument schema summary must be included"
|
||||
|
||||
|
||||
def test_find_capability_no_match_suggests_full_registry():
|
||||
executor = ltr._find_capability_executor
|
||||
result = executor("find_capability", {"query": "zzzqqq xxx"})
|
||||
assert result.get("success") is True
|
||||
assert "find_capability" in result.get("result", "")
|
||||
|
||||
|
||||
def test_runtime_counts_find_capability_calls():
|
||||
"""find_capability round-trips are visible to the trace (v3 field)."""
|
||||
responses = [
|
||||
_record_tool("find_capability", {"query": "commit"}, call_id="c1"),
|
||||
{"content": "nothing to commit"},
|
||||
]
|
||||
transport = _FakeTransport(responses)
|
||||
runtime = LocalToolRuntime(
|
||||
transport=transport,
|
||||
executor=ltr._find_capability_executor,
|
||||
model="m",
|
||||
tools=cb.select_for_job("fix code")["tools"],
|
||||
)
|
||||
result = runtime.run([{"role": "user", "content": "commit"}])
|
||||
assert result.success is True
|
||||
assert getattr(runtime, "find_capability_calls", -1) == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 4 — composition registration + authority map + docs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_composition_registers_capability_selection():
|
||||
from rune.orchestration.composition import (
|
||||
AUTHORITY_KINDS,
|
||||
CapabilitySelectionAuthority,
|
||||
CompositionRoot,
|
||||
compose_capability_selection,
|
||||
)
|
||||
|
||||
assert "capability_selection" in AUTHORITY_KINDS
|
||||
root = CompositionRoot()
|
||||
authority = compose_capability_selection(root, tmp_state := Path(_tmpdir()))
|
||||
registered = root.get("capability_selection")
|
||||
assert registered is not None
|
||||
assert authority.broker is not None
|
||||
selection = authority.select_for_job("fix the bug in parse.py")
|
||||
assert selection["names"]
|
||||
|
||||
|
||||
def _tmpdir():
|
||||
import tempfile
|
||||
|
||||
return tempfile.mkdtemp(prefix="p1-01-compose-")
|
||||
|
||||
|
||||
def test_composition_single_authority_enforced():
|
||||
from rune.orchestration.composition import (
|
||||
DuplicateAuthorityError,
|
||||
compose_capability_selection,
|
||||
)
|
||||
from rune.orchestration.composition import CompositionRoot
|
||||
|
||||
root = CompositionRoot()
|
||||
compose_capability_selection(root, Path(_tmpdir()))
|
||||
compose_capability_selection(root, Path(_tmpdir()))
|
||||
try:
|
||||
root.assert_single_authority()
|
||||
raise AssertionError("expected DuplicateAuthorityError")
|
||||
except DuplicateAuthorityError:
|
||||
pass
|
||||
|
||||
|
||||
def test_authority_map_capability_selection_is_live():
|
||||
manifest = json.loads((ROOT / "config" / "v1_authority_map.json").read_text())
|
||||
responsibility = next(r for r in manifest["responsibilities"] if r["id"] == "capability_selection")
|
||||
paths = {p["path"].split(":")[0]: p for p in responsibility["current_paths"]}
|
||||
assert "disposition" in paths["src/rune/capability_broker.py"]
|
||||
assert paths["src/rune/capability_broker.py"]["disposition"] == "retain-authority"
|
||||
assert "select_for_job" in paths["src/rune/capability_broker.py"]["evidence"]
|
||||
assert paths["memory/index/capability_cards.json"]["disposition"] == "migrate-adapter"
|
||||
assert "src/rune/local_tool_runtime.py" in paths
|
||||
assert paths["src/rune/local_tool_runtime.py"]["disposition"] == "migrate-adapter"
|
||||
|
||||
|
||||
def test_capability_selection_doc_exists():
|
||||
doc = ROOT / "docs" / "capability-selection.md"
|
||||
assert doc.is_file()
|
||||
text = doc.read_text()
|
||||
assert "select_for_job" in text
|
||||
assert "find_capability" in text
|
||||
assert "FLOOR_TOOLS" in text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality discharges (post-review findings 1-7)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
import inspect # noqa: E402
|
||||
|
||||
|
||||
def test_load_cards_is_defined_once():
|
||||
"""Finding 1: the byte-identical duplicate ``load_cards`` (F811) is
|
||||
deleted — exactly one definition survives in the broker module."""
|
||||
source = inspect.getsource(cb)
|
||||
assert source.count("def load_cards") == 1, "load_cards must be defined exactly once (duplicate shadows the first)"
|
||||
|
||||
|
||||
def test_every_tool_schema_is_a_valid_json_object_schema():
|
||||
"""Finding 3: every tool's ``parameters`` must be a valid JSON schema
|
||||
object shape — ``{}`` (read_persona) is not a schema."""
|
||||
for tool in TOOLS:
|
||||
params = tool.get("parameters")
|
||||
assert isinstance(params, dict), f"{tool['name']}: parameters must be a dict"
|
||||
assert params.get("type") == "object", f"{tool['name']}: parameters.type must be 'object'"
|
||||
assert isinstance(params.get("properties"), dict), f"{tool['name']}: parameters.properties"
|
||||
|
||||
|
||||
def test_hatch_dispatch_routes_through_validated_dispatch(tmp_path, monkeypatch):
|
||||
"""Finding 4: find_capability rides dispatch_validated_tool like every
|
||||
other tool (was: a direct executor call that bypassed validation)."""
|
||||
real_dispatch = ltr.dispatch_validated_tool
|
||||
seen: list[str] = []
|
||||
|
||||
def spy(request, executor):
|
||||
seen.append(request.name)
|
||||
return real_dispatch(request, executor)
|
||||
|
||||
monkeypatch.setattr(ltr, "dispatch_validated_tool", spy)
|
||||
|
||||
responses = [
|
||||
_record_tool("find_capability", {"query": "map the repository structure"}, call_id="c1"),
|
||||
{"content": "done"},
|
||||
]
|
||||
runtime = LocalToolRuntime(
|
||||
transport=_FakeTransport(responses),
|
||||
executor=lambda n, a: {"success": True, "result": "ok"}, # noqa: E731
|
||||
model="m",
|
||||
tools=cb.select_for_job("fix code")["tools"],
|
||||
)
|
||||
result = runtime.run([{"role": "user", "content": "go"}])
|
||||
assert result.success is True, result.error
|
||||
assert "find_capability" in seen, "hatch must dispatch through the validated path"
|
||||
assert runtime.find_capability_calls == 1
|
||||
# Exposure still flows: the discovered schema joined the runtime set.
|
||||
assert "repo_map" in runtime.tool_names
|
||||
|
||||
|
||||
def test_resume_honors_exposed_tools_in_pending_state(tmp_path, monkeypatch):
|
||||
"""Finding 5: the runtime-built pending_state carries the run's exposed
|
||||
tool set, and a FRESH runtime instance can resume it. Real runtime +
|
||||
real broker query — only the LLM transport is scripted."""
|
||||
cards = tmp_path / "capability_cards.json"
|
||||
cards.write_text("[]")
|
||||
monkeypatch.setattr(cb, "CARDS_FILE", str(cards))
|
||||
monkeypatch.setattr(cb, "REGISTRY", str(tmp_path / "no-registry.json"))
|
||||
|
||||
executor = lambda n, a: {"success": True, "result": "ok"} # noqa: E731
|
||||
transport = _FakeTransport(
|
||||
[
|
||||
_record_tool("find_capability", {"query": "commit my changes"}, call_id="c1"),
|
||||
_record_tool("git_commit", {"message": "m"}, call_id="c2"),
|
||||
]
|
||||
)
|
||||
runtime = LocalToolRuntime(
|
||||
transport=transport,
|
||||
executor=executor,
|
||||
model="m",
|
||||
tools=cb.select_for_job("fix code")["tools"],
|
||||
)
|
||||
response = runtime.run([{"role": "user", "content": "commit"}])
|
||||
assert response.pending_approval is not None, "git_commit must gate on approval"
|
||||
assert response.pending_approval.name == "git_commit"
|
||||
state = response.pending_state
|
||||
exposed = state.get("exposed_tools")
|
||||
assert isinstance(exposed, list) and "git_commit" in exposed, (
|
||||
"runtime-built pending_state must carry the dynamically exposed tool"
|
||||
)
|
||||
assert "find_capability" in exposed
|
||||
|
||||
fresh = LocalToolRuntime(
|
||||
transport=_FakeTransport([{"content": "committed"}]),
|
||||
executor=executor,
|
||||
model="m",
|
||||
tools=cb.select_for_job("fix code")["tools"],
|
||||
)
|
||||
result = fresh.resume(state, {"success": True, "result": "committed"})
|
||||
assert result.success is True, result.error
|
||||
assert result.user_text == "committed"
|
||||
|
||||
|
||||
def test_capability_selection_degrade_reason_propagates(tmp_path):
|
||||
"""Finding 6a: operator-level degrade must never produce an empty
|
||||
capability_reason while selecting the full toolset."""
|
||||
storage = tmp_path / "operator-degrade.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
|
||||
def _boom(description, need_tools=True):
|
||||
raise RuntimeError("broker gone")
|
||||
|
||||
with mock.patch.object(cb, "select_for_job", _boom):
|
||||
selection = op._capability_selection("fix the bug in parse.py")
|
||||
assert [t["name"] for t in selection["tools"]] == [t["name"] for t in TOOLS]
|
||||
assert selection["reason"] == "fallback full toolset (broker unavailable)"
|
||||
|
||||
|
||||
def test_operator_run_selects_capabilities_once(tmp_path, monkeypatch):
|
||||
"""Finding 6b: one ``_capability_selection`` call per run (was three:
|
||||
runtime construction + trace selected_tools + trace capability_reason)."""
|
||||
monkeypatch.setenv("RUNE_HOME", str(tmp_path / "home-q"))
|
||||
(tmp_path / "home-q" / "state").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
storage = tmp_path / "operator-q.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
trace = {
|
||||
"model_id": "m",
|
||||
"window": 40960,
|
||||
"budget": 32768,
|
||||
"blocks": [
|
||||
{"name": "persona", "tokens": 10, "admitted": True, "reason": ""},
|
||||
{"name": "user", "tokens": 5, "admitted": True, "reason": ""},
|
||||
],
|
||||
"admitted_total": 15,
|
||||
"dropped": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"compaction": [],
|
||||
"elapsed_ms": 1.0,
|
||||
}
|
||||
capsule = mock.Mock(
|
||||
token_count=15,
|
||||
active_objective="",
|
||||
blocks=[],
|
||||
to_messages=lambda: [{"role": "user", "content": "do it"}],
|
||||
)
|
||||
broker = mock.Mock(assemble=lambda task, session, model_id: (capsule, trace))
|
||||
response = rune_operator.OperatorResponse(success=True, user_text="done", model="m")
|
||||
|
||||
class _RT:
|
||||
transport = mock.Mock(provider="ollama")
|
||||
tool_names = frozenset({"read_file", "find_capability"})
|
||||
|
||||
def run(self, messages, **kwargs):
|
||||
return response
|
||||
|
||||
with (
|
||||
mock.patch.object(op, "_context_broker", return_value=broker),
|
||||
mock.patch.object(rune_operator.model_profiles, "get_or_create_profile", return_value=mock.Mock(safety_tier=0)),
|
||||
mock.patch.object(rune_operator.model_profiles, "record_interaction"),
|
||||
mock.patch.object(rune_operator, "LocalToolRuntime", return_value=_RT()),
|
||||
mock.patch.object(rune_operator, "OllamaChatTransport", return_value=mock.Mock(provider="ollama")),
|
||||
mock.patch("rune.local_tool_runtime.context_length", lambda model_: 40960),
|
||||
mock.patch.object(
|
||||
rune_operator.safety_override.SafetyOverride,
|
||||
"get_params",
|
||||
lambda self, text: {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""},
|
||||
),
|
||||
mock.patch.object(rune_operator.safety_override.SafetyOverride, "is_refusal", lambda self, text: False),
|
||||
mock.patch.object(op, "_log_token_usage"),
|
||||
mock.patch("rune.event_bus._store_path", lambda: tmp_path / "journal-q-events.jsonl"),
|
||||
mock.patch.object(op, "_capability_selection", side_effect=op._capability_selection) as spy,
|
||||
):
|
||||
envelope = op.send_task("fix this code path")
|
||||
|
||||
assert envelope.get("success") is True
|
||||
assert spy.call_count == 1, f"_capability_selection must run once per run, got {spy.call_count}"
|
||||
|
||||
|
||||
def test_load_cards_rebuild_write_is_atomic(tmp_path, monkeypatch):
|
||||
"""Finding 7: the rebuild persists via temp file + os.replace — never a
|
||||
direct in-place write that a crash could truncate into corrupt JSON."""
|
||||
cards_file = tmp_path / "capability_cards.json"
|
||||
cards_file.write_text("[]")
|
||||
monkeypatch.setattr(cb, "CARDS_FILE", str(cards_file))
|
||||
monkeypatch.setattr(cb, "REGISTRY", str(tmp_path / "no-registry.json"))
|
||||
|
||||
real_replace = os.replace
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
def spy(src, dst, **kwargs):
|
||||
calls.append((str(src), str(dst)))
|
||||
return real_replace(src, dst, **kwargs)
|
||||
|
||||
monkeypatch.setattr(os, "replace", spy)
|
||||
loaded = cb.load_cards()
|
||||
assert any(card.get("native_tool") for card in loaded)
|
||||
assert len(calls) == 1, f"expected exactly one atomic rename, got {calls}"
|
||||
src, dst = calls[0]
|
||||
assert dst == str(cards_file)
|
||||
assert src != str(cards_file), "must write a temp file, not the index in place"
|
||||
assert src.startswith(str(tmp_path)), "temp file must live beside the index"
|
||||
assert json.loads(cards_file.read_text()) == loaded
|
||||
@@ -1,523 +0,0 @@
|
||||
"""P1-02 (#25): AttentionCapsule authority — FROZEN TESTS ARE THE SPEC.
|
||||
|
||||
Layers covered per plan task:
|
||||
1. ContributionBlock.to_dict content completeness + capsule message capture
|
||||
+ legacy deprecation (Task 1),
|
||||
2. safety-override re-measure + num_ctx budget propagation (Task 2),
|
||||
3. trace v4 capsule fidelity fields + operator logging (Task 3).
|
||||
|
||||
The canary (src/rune/canary_p1_02.py) re-proves the contract CI-side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
from rune import rune_operator # noqa: E402
|
||||
from rune.attention_capsule import AttentionCapsule # noqa: E402
|
||||
from rune.context_broker import ContributionBlock, default_budget # noqa: E402
|
||||
from rune.local_tool_runtime import OllamaChatTransport # noqa: E402
|
||||
from rune.tool_protocol import OperatorResponse # noqa: E402
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _ScriptedTransport(OllamaChatTransport):
|
||||
"""OllamaChatTransport with _post scripted — real accumulation code runs."""
|
||||
|
||||
def __init__(self, responses):
|
||||
super().__init__()
|
||||
self._responses = list(responses)
|
||||
|
||||
def _post(self, path, payload):
|
||||
if not self._responses:
|
||||
raise AssertionError("scripted transport exhausted")
|
||||
return self._responses.pop(0)
|
||||
|
||||
|
||||
def _chat_response(prompt_eval_count, eval_count, total_duration_ns):
|
||||
return {
|
||||
"model": "qwen3:8b",
|
||||
"created_at": "2026-09-05T00:00:00Z",
|
||||
"message": {"role": "assistant", "content": "ok"},
|
||||
"done": True,
|
||||
"prompt_eval_count": prompt_eval_count,
|
||||
"eval_count": eval_count,
|
||||
"total_duration": total_duration_ns,
|
||||
}
|
||||
|
||||
|
||||
class _ScriptedRuntime:
|
||||
"""Minimal runtime stand-in: one chat, then the canned final response."""
|
||||
|
||||
def __init__(self, transport, response):
|
||||
self.transport = transport
|
||||
self.tool_names = ("read_file", "write_file")
|
||||
self._response = response
|
||||
self.received_messages: list = []
|
||||
self.received_options: dict = {}
|
||||
|
||||
def run(self, messages, **kwargs):
|
||||
self.received_messages = [dict(m) for m in messages]
|
||||
self.received_options = dict(kwargs.get("options") or {})
|
||||
self.transport.chat(model="qwen3:8b", messages=list(messages), tools=None)
|
||||
return self._response
|
||||
|
||||
|
||||
def _broker_trace(**overrides):
|
||||
trace = {
|
||||
"model_id": "qwen3:8b",
|
||||
"window": 40960,
|
||||
"budget": 32768,
|
||||
"blocks": [
|
||||
{"name": "persona", "tokens": 980, "admitted": True, "reason": ""},
|
||||
{"name": "user", "tokens": 20, "admitted": True, "reason": ""},
|
||||
],
|
||||
"admitted": ["persona", "user"],
|
||||
"admitted_total": 1000,
|
||||
"dropped": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"compaction": [],
|
||||
"elapsed_ms": 1.0,
|
||||
}
|
||||
trace.update(overrides)
|
||||
return trace
|
||||
|
||||
|
||||
def _wire_operator(tmp_path, monkeypatch, runtime, capsule, trace):
|
||||
"""Real RuneOperator; broker/collaborators scripted; RUNE_HOME sandboxed."""
|
||||
monkeypatch.setenv("RUNE_HOME", str(tmp_path))
|
||||
op = rune_operator.RuneOperator(str(tmp_path / "operator.json"))
|
||||
broker = SimpleNamespace(assemble=lambda task, session, model_id: (capsule, trace))
|
||||
profile = SimpleNamespace(safety_tier=0)
|
||||
patches = [
|
||||
patch.object(op, "_context_broker", return_value=broker),
|
||||
patch.object(rune_operator.model_profiles, "get_or_create_profile", return_value=profile),
|
||||
patch.object(rune_operator.model_profiles, "record_interaction"),
|
||||
patch.object(rune_operator, "LocalToolRuntime", return_value=runtime),
|
||||
patch.object(rune_operator, "OllamaChatTransport", return_value=runtime.transport),
|
||||
patch("rune.local_tool_runtime.context_length", lambda model: 40960),
|
||||
patch.object(
|
||||
rune_operator.safety_override.SafetyOverride,
|
||||
"get_params",
|
||||
lambda self, text: {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""},
|
||||
),
|
||||
patch.object(rune_operator.safety_override.SafetyOverride, "is_refusal", lambda self, text: False),
|
||||
patch.object(op, "_log_token_usage"),
|
||||
patch("rune.event_bus._store_path", lambda: tmp_path / "journal-events.jsonl"),
|
||||
]
|
||||
return op, patches
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 1 — ContributionBlock.to_dict content + capsule message capture
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_contribution_block_to_dict_includes_content():
|
||||
"""to_dict must carry the block's content so pending-state/log capsules
|
||||
are verifiable (P1-02 acceptance: logged capsule contents == payload)."""
|
||||
block = ContributionBlock("user", 100, "inspect this", 5, "rune_operator task text", evictable=False)
|
||||
serialized = block.to_dict()
|
||||
assert serialized["content"] == "inspect this"
|
||||
assert serialized["name"] == "user"
|
||||
assert serialized["priority"] == 100
|
||||
assert serialized["tokens"] == 5
|
||||
assert serialized["source"] == "rune_operator task text"
|
||||
assert serialized["evictable"] is False
|
||||
|
||||
|
||||
def test_contribution_block_to_dict_round_trips_list_content():
|
||||
"""List-content (history) blocks serialize the message list verbatim."""
|
||||
messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]
|
||||
block = ContributionBlock("history", 30, messages, 10, "history")
|
||||
assert block.to_dict()["content"] == messages
|
||||
|
||||
|
||||
def test_capsule_to_dict_embeds_block_content():
|
||||
"""Capsule to_dict round-trips the block content through the ledger."""
|
||||
capsule = AttentionCapsule(token_count=5)
|
||||
capsule.blocks = [ContributionBlock("user", 100, "do the thing", 3, "src", evictable=False)]
|
||||
data = capsule.to_dict()
|
||||
assert data["blocks"][0]["content"] == "do the thing"
|
||||
assert json.loads(json.dumps(data)) == data # JSON-safe
|
||||
|
||||
|
||||
def test_capture_messages_stores_the_exact_list():
|
||||
"""capture_messages() stores the sent list post-hoc (dynamic attr like
|
||||
messages/system_preamble) — no dataclass field, additive only."""
|
||||
capsule = AttentionCapsule()
|
||||
capsule.blocks = [
|
||||
ContributionBlock("persona", 90, "I am Rune.", 2, "persona", evictable=False),
|
||||
ContributionBlock("user", 100, "go", 1, "user", evictable=False),
|
||||
]
|
||||
sent = capsule.to_messages()
|
||||
returned = capsule.capture_messages(sent)
|
||||
assert returned == sent
|
||||
assert capsule.sent_messages == sent
|
||||
# capture after in-place mutation re-captures the mutated rendering
|
||||
capsule.blocks[-1].content = "decorated go"
|
||||
assert capsule.to_messages() != sent
|
||||
capsule.capture_messages(capsule.to_messages())
|
||||
assert capsule.sent_messages == capsule.to_messages()
|
||||
|
||||
|
||||
def test_capture_messages_copies_so_later_mutation_does_not_diverge():
|
||||
"""The stored list must not alias the caller's list (log integrity)."""
|
||||
capsule = AttentionCapsule()
|
||||
sent = [{"role": "user", "content": "go"}]
|
||||
capsule.capture_messages(sent)
|
||||
sent.append({"role": "assistant", "content": "hijack"})
|
||||
assert capsule.sent_messages == [{"role": "user", "content": "go"}]
|
||||
|
||||
|
||||
def test_to_messages_rederivation_equals_captured_list():
|
||||
"""Equality invariant: to_messages() re-derivation equals the captured
|
||||
sent list for the same capsule state."""
|
||||
capsule = AttentionCapsule()
|
||||
capsule.blocks = [
|
||||
ContributionBlock("persona", 90, "I am Rune.", 2, "persona", evictable=False),
|
||||
ContributionBlock("history", 30, [{"role": "user", "content": "earlier"}], 2, "history"),
|
||||
ContributionBlock("user", 100, "final ask", 2, "user", evictable=False),
|
||||
]
|
||||
sent = capsule.to_messages()
|
||||
capsule.capture_messages(sent)
|
||||
assert capsule.to_messages() == capsule.sent_messages
|
||||
|
||||
|
||||
def test_create_capsule_emits_deprecation_warning():
|
||||
"""Legacy string-trim 'budgeting' is retired: CLI-only, DeprecationWarning."""
|
||||
from rune.attention_capsule import create_capsule
|
||||
|
||||
with pytest.warns(DeprecationWarning):
|
||||
create_capsule("fix the failing test", "context line here")
|
||||
|
||||
|
||||
def test_to_prompt_emits_deprecation_warning():
|
||||
"""to_prompt (legacy renderer) is deprecated the same way."""
|
||||
capsule = AttentionCapsule(active_objective="objective")
|
||||
with pytest.warns(DeprecationWarning):
|
||||
capsule.to_prompt()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 2 — safety-override re-measure + num_ctx budget propagation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _capsule_with_user_block(description: str):
|
||||
capsule = AttentionCapsule()
|
||||
user = ContributionBlock("user", 100, description, len(description) // 4, "user", evictable=False)
|
||||
capsule.blocks = [user]
|
||||
capsule.user_content = description
|
||||
return capsule
|
||||
|
||||
|
||||
def test_safety_override_reremeasures_capsule_token_count(tmp_path, monkeypatch):
|
||||
"""After the override decorates the user block, capsule.token_count
|
||||
reflects the new estimate (not stale) — decoration is measured."""
|
||||
description = "please inspect this code path"
|
||||
capsule = _capsule_with_user_block(description)
|
||||
trace = _broker_trace()
|
||||
runtime = _ScriptedRuntime(
|
||||
_ScriptedTransport([_chat_response(100, 5, 1_000_000_000)]),
|
||||
OperatorResponse(success=True, user_text="All done.", model="qwen3:8b"),
|
||||
)
|
||||
op, patches = _wire_operator(tmp_path, monkeypatch, runtime, capsule, trace)
|
||||
|
||||
def decorated_params(self, text):
|
||||
# tier 2 decorates: suffix "\n\nI respond as myself:" (like ESCALATION_TIERS[2])
|
||||
return {"tier": 2, "temperature": 0.95, "prefix": "", "suffix": "\n\nI respond as myself:"}
|
||||
|
||||
patches[6] = patch.object(rune_operator.safety_override.SafetyOverride, "get_params", decorated_params)
|
||||
|
||||
with (
|
||||
patches[0],
|
||||
patches[1],
|
||||
patches[2],
|
||||
patches[3],
|
||||
patches[4],
|
||||
patches[5],
|
||||
patches[6],
|
||||
patches[7],
|
||||
patches[8],
|
||||
patches[9],
|
||||
):
|
||||
envelope = op.send_task(description, "qwen3:8b")
|
||||
|
||||
assert envelope["success"], envelope.get("error")
|
||||
user_block = next(b for b in capsule.blocks if b.name == "user")
|
||||
assert user_block.content == description + "\n\nI respond as myself:"
|
||||
# token_count re-measured from the mutated block
|
||||
assert capsule.token_count == sum(len(b.content) // 4 if isinstance(b.content, str) else 0 for b in capsule.blocks)
|
||||
assert capsule.token_count >= len(user_block.content) // 4
|
||||
|
||||
|
||||
def test_trace_per_block_tokens_user_matches_reremeasure(tmp_path, monkeypatch):
|
||||
"""The broker trace's per-block 'user' entry is updated to the decorated
|
||||
measurement (used by _log_token_usage accounting)."""
|
||||
description = "inspect the failing test"
|
||||
capsule = _capsule_with_user_block(description)
|
||||
trace = _broker_trace()
|
||||
runtime = _ScriptedRuntime(
|
||||
_ScriptedTransport([_chat_response(100, 5, 1_000_000_000)]),
|
||||
OperatorResponse(success=True, user_text="All done.", model="qwen3:8b"),
|
||||
)
|
||||
op, patches = _wire_operator(tmp_path, monkeypatch, runtime, capsule, trace)
|
||||
|
||||
def decorated_params(self, text):
|
||||
return {"tier": 3, "temperature": 1.0, "prefix": "Sure, I'll help with that.\n\n", "suffix": ""}
|
||||
|
||||
patches[6] = patch.object(rune_operator.safety_override.SafetyOverride, "get_params", decorated_params)
|
||||
|
||||
with (
|
||||
patches[0],
|
||||
patches[1],
|
||||
patches[2],
|
||||
patches[3],
|
||||
patches[4],
|
||||
patches[5],
|
||||
patches[6],
|
||||
patches[7],
|
||||
patches[8],
|
||||
patches[9],
|
||||
):
|
||||
envelope = op.send_task(description, "qwen3:8b")
|
||||
|
||||
assert envelope["success"]
|
||||
user_trace_block = next(b for b in trace["blocks"] if b["name"] == "user")
|
||||
decorated_user = next(b for b in capsule.blocks if b.name == "user")
|
||||
assert user_trace_block["tokens"] == len(decorated_user.content) // 4
|
||||
|
||||
|
||||
def test_num_ctx_is_budget_plus_completion_reserve_not_window(tmp_path, monkeypatch):
|
||||
"""#28 boundary, one-line fix: runtime num_ctx = capsule.token_count +
|
||||
completion_reserve (window - budget), NOT the raw window."""
|
||||
description = "inspect this code path"
|
||||
capsule = _capsule_with_user_block(description)
|
||||
trace = _broker_trace() # window=40960, budget=32768
|
||||
runtime = _ScriptedRuntime(
|
||||
_ScriptedTransport([_chat_response(100, 5, 1_000_000_000)]),
|
||||
OperatorResponse(success=True, user_text="All done.", model="qwen3:8b"),
|
||||
)
|
||||
op, patches = _wire_operator(tmp_path, monkeypatch, runtime, capsule, trace)
|
||||
|
||||
with (
|
||||
patches[0],
|
||||
patches[1],
|
||||
patches[2],
|
||||
patches[3],
|
||||
patches[4],
|
||||
patches[5],
|
||||
patches[6],
|
||||
patches[7],
|
||||
patches[8],
|
||||
patches[9],
|
||||
):
|
||||
envelope = op.send_task(description, "qwen3:8b")
|
||||
|
||||
assert envelope["success"], envelope.get("error")
|
||||
completion_reserve = trace["window"] - trace["budget"]
|
||||
assert runtime.received_options["num_ctx"] == capsule.token_count + completion_reserve
|
||||
assert runtime.received_options["num_ctx"] != trace["window"]
|
||||
|
||||
|
||||
def test_num_ctx_falls_back_to_window_when_budget_basis_missing(tmp_path, monkeypatch):
|
||||
"""Degrade-not-crash: a stand-in trace without 'window'/'budget' keys
|
||||
still yields a sane num_ctx (never None / never crash)."""
|
||||
description = "inspect this code path"
|
||||
capsule = _capsule_with_user_block(description)
|
||||
trace = _broker_trace()
|
||||
trace.pop("window")
|
||||
trace.pop("budget")
|
||||
runtime = _ScriptedRuntime(
|
||||
_ScriptedTransport([_chat_response(100, 5, 1_000_000_000)]),
|
||||
OperatorResponse(success=True, user_text="All done.", model="qwen3:8b"),
|
||||
)
|
||||
op, patches = _wire_operator(tmp_path, monkeypatch, runtime, capsule, trace)
|
||||
|
||||
with (
|
||||
patches[0],
|
||||
patches[1],
|
||||
patches[2],
|
||||
patches[3],
|
||||
patches[4],
|
||||
patches[5],
|
||||
patches[6],
|
||||
patches[7],
|
||||
patches[8],
|
||||
patches[9],
|
||||
):
|
||||
envelope = op.send_task(description, "qwen3:8b")
|
||||
|
||||
assert envelope["success"], envelope.get("error")
|
||||
assert isinstance(runtime.received_options.get("num_ctx"), int)
|
||||
assert runtime.received_options["num_ctx"] > 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 3 — trace v4 capsule fidelity fields + operator logging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_inference_trace_schema_v4_fields():
|
||||
"""Additive v4: capsule_messages / capsule_content_hashes /
|
||||
capsule_token_count with defaults; v1-v3 readers unaffected."""
|
||||
from rune import inference_trace as it
|
||||
|
||||
assert it.INFERENCE_TRACE_SCHEMA_VERSION == 4
|
||||
trace = it.InferenceTrace(
|
||||
schema_version=it.INFERENCE_TRACE_SCHEMA_VERSION,
|
||||
provider="ollama",
|
||||
model="m",
|
||||
task_id="t",
|
||||
channel="c",
|
||||
resolved_context_window=0,
|
||||
input_budget=0,
|
||||
model_calls=0,
|
||||
prompt_tokens_estimated=0,
|
||||
prompt_tokens_served=0,
|
||||
output_tokens=0,
|
||||
latency_ms=0.0,
|
||||
model_time_ms=0.0,
|
||||
per_block_tokens={},
|
||||
dropped_blocks=[],
|
||||
tools_exposed=[],
|
||||
memories_exposed=[],
|
||||
memory_tokens=0,
|
||||
degraded=[],
|
||||
cache=None,
|
||||
cost_usd=None,
|
||||
cost_basis="local-unpriced",
|
||||
success=True,
|
||||
error="",
|
||||
)
|
||||
assert trace.capsule_messages == []
|
||||
assert trace.capsule_content_hashes == {}
|
||||
assert trace.capsule_token_count == 0
|
||||
|
||||
|
||||
def test_trace_v1_row_without_v4_fields_still_parses(tmp_path):
|
||||
from rune import inference_trace as it
|
||||
|
||||
row = {
|
||||
"schema_version": 1,
|
||||
"provider": "ollama",
|
||||
"model": "m",
|
||||
"task_id": "t",
|
||||
"channel": "c",
|
||||
"resolved_context_window": 0,
|
||||
"input_budget": 0,
|
||||
"model_calls": 0,
|
||||
"prompt_tokens_estimated": 0,
|
||||
"prompt_tokens_served": 0,
|
||||
"output_tokens": 0,
|
||||
"latency_ms": 0.0,
|
||||
"model_time_ms": 0.0,
|
||||
"per_block_tokens": {},
|
||||
"dropped_blocks": [],
|
||||
"tools_exposed": [],
|
||||
"memories_exposed": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"cache": None,
|
||||
"cost_usd": None,
|
||||
"cost_basis": "local-unpriced",
|
||||
"success": True,
|
||||
"error": "",
|
||||
}
|
||||
path = tmp_path / "inference_traces.jsonl"
|
||||
path.write_text(json.dumps(row) + "\n", encoding="utf-8")
|
||||
records = it.read_traces(trace_dir=tmp_path)
|
||||
assert records == [row] # readers never require v4 keys
|
||||
|
||||
|
||||
def test_operator_trace_record_carries_capsule_messages(tmp_path, monkeypatch):
|
||||
"""The durable trace logs the EXACT messages the runtime received —
|
||||
the acceptance criterion, machine-checked."""
|
||||
description = "inspect this code path"
|
||||
capsule = _capsule_with_user_block(description)
|
||||
trace = _broker_trace()
|
||||
runtime = _ScriptedRuntime(
|
||||
_ScriptedTransport([_chat_response(100, 5, 1_000_000_000)]),
|
||||
OperatorResponse(success=True, user_text="All done.", model="qwen3:8b"),
|
||||
)
|
||||
op, patches = _wire_operator(tmp_path, monkeypatch, runtime, capsule, trace)
|
||||
|
||||
with (
|
||||
patches[0],
|
||||
patches[1],
|
||||
patches[2],
|
||||
patches[3],
|
||||
patches[4],
|
||||
patches[5],
|
||||
patches[6],
|
||||
patches[7],
|
||||
patches[8],
|
||||
patches[9],
|
||||
):
|
||||
envelope = op.send_task(description, "qwen3:8b")
|
||||
|
||||
assert envelope["success"], envelope.get("error")
|
||||
from rune import inference_trace as it
|
||||
|
||||
records = it.read_traces(trace_dir=tmp_path / "state")
|
||||
assert records, "durable trace missing"
|
||||
record = records[-1]
|
||||
assert record["capsule_messages"] == runtime.received_messages, (
|
||||
"logged capsule messages must EXACTLY equal what runtime.run received"
|
||||
)
|
||||
assert record["capsule_messages"][-1]["content"] == description
|
||||
assert record["capsule_token_count"] == capsule.token_count
|
||||
|
||||
|
||||
def test_operator_trace_record_carries_content_hashes(tmp_path, monkeypatch):
|
||||
"""Per-block SHA-256 of content — integrity without JSONL bloat."""
|
||||
description = "inspect this code path"
|
||||
capsule = _capsule_with_user_block(description)
|
||||
trace = _broker_trace()
|
||||
runtime = _ScriptedRuntime(
|
||||
_ScriptedTransport([_chat_response(100, 5, 1_000_000_000)]),
|
||||
OperatorResponse(success=True, user_text="All done.", model="qwen3:8b"),
|
||||
)
|
||||
op, patches = _wire_operator(tmp_path, monkeypatch, runtime, capsule, trace)
|
||||
|
||||
with (
|
||||
patches[0],
|
||||
patches[1],
|
||||
patches[2],
|
||||
patches[3],
|
||||
patches[4],
|
||||
patches[5],
|
||||
patches[6],
|
||||
patches[7],
|
||||
patches[8],
|
||||
patches[9],
|
||||
):
|
||||
envelope = op.send_task(description, "qwen3:8b")
|
||||
|
||||
assert envelope["success"]
|
||||
from rune import inference_trace as it
|
||||
|
||||
record = it.read_traces(trace_dir=tmp_path / "state")[-1]
|
||||
for block in capsule.blocks:
|
||||
expected = hashlib.sha256(
|
||||
block.content.encode("utf-8")
|
||||
if isinstance(block.content, str)
|
||||
else json.dumps(block.content).encode("utf-8")
|
||||
).hexdigest()
|
||||
assert record["capsule_content_hashes"][block.name] == expected
|
||||
@@ -1,568 +0,0 @@
|
||||
"""P1-06 model/provider routing — frozen tests (the spec, issue #29).
|
||||
|
||||
Task order mirrors plans/2026-09-06-p1-06-model-routing.md:
|
||||
1. ProviderAdapter Protocol + OllamaAdapter
|
||||
2. Measured registry + ModelRouter.rank()
|
||||
3. Operator wiring + escalation start-tier + trace fields
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fakes (scripted — no network, no live Ollama)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeTransport:
|
||||
"""Scripted transport double: records chat calls, returns canned dicts."""
|
||||
|
||||
def __init__(self, expected: dict | None = None):
|
||||
self.expected = expected or {"message": {"role": "assistant", "content": "ok"}}
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def available(self) -> bool:
|
||||
return True
|
||||
|
||||
def chat(self, *, model, messages, tools=None, options=None, think=None):
|
||||
self.calls.append(
|
||||
{
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"tools": tools,
|
||||
"options": options,
|
||||
"think": think,
|
||||
}
|
||||
)
|
||||
return self.expected
|
||||
|
||||
|
||||
class _CloudStubAdapter:
|
||||
"""Declared-but-not-live cloud adapter (imported from the module)."""
|
||||
|
||||
def __new__(cls):
|
||||
from rune.model_routing import CloudStubAdapter, ModelRouter, MeasuredModelProfile, load_profiles, save_profiles
|
||||
|
||||
return super().__new__(CloudStubAdapter)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 1 — ProviderAdapter Protocol + OllamaAdapter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_provider_adapter_protocol_shape():
|
||||
from rune.model_routing import OllamaAdapter, ProviderUnavailable, ProviderAdapter
|
||||
|
||||
a = OllamaAdapter(transport=_FakeTransport())
|
||||
assert isinstance(a, ProviderAdapter)
|
||||
assert a.name == "ollama"
|
||||
with pytest.raises(ProviderUnavailable):
|
||||
_CloudStubAdapter().chat(model="x", messages=[], tools=None, options={}, think=False)
|
||||
|
||||
|
||||
def test_ollama_adapter_delegates_to_transport():
|
||||
from rune.model_routing import OllamaAdapter
|
||||
|
||||
a = OllamaAdapter(transport=_FakeTransport(expected={"message": {"content": "hi"}}))
|
||||
resp = a.chat(
|
||||
model="qwen3:8b",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
tools=None,
|
||||
options={},
|
||||
think=False,
|
||||
)
|
||||
assert resp["message"]["content"] == "hi"
|
||||
assert a.available() is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 2 — Measured registry + ModelRouter.rank()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _profiles_file(tmp_path, rows):
|
||||
path = tmp_path / "state" / "model_routing_profiles.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(rows))
|
||||
return path
|
||||
|
||||
|
||||
def test_rank_respects_capability_floor_and_tool_need():
|
||||
from rune.model_routing import ModelRouter
|
||||
|
||||
router = ModelRouter() # no profiles: capability floor only
|
||||
ranked = router.rank(task_family="code", need_tools=True)
|
||||
# T1..T3 do not support tools; only T4 (cloud, declared) passes the floor
|
||||
assert [d.model for d in ranked] == ["mimo-v2.5-pro"]
|
||||
assert ranked[0].reason.startswith("no-measured-profiles")
|
||||
|
||||
|
||||
def test_rank_no_measured_profiles_reason():
|
||||
from rune.model_routing import ModelRouter
|
||||
|
||||
ranked = ModelRouter().rank(task_family="code")
|
||||
assert ranked, "capability-floor must always yield candidates"
|
||||
assert all(d.reason.startswith("no-measured-profiles") for d in ranked)
|
||||
|
||||
|
||||
def test_rank_cheapest_verified_capable_first_with_measured_reason(tmp_path):
|
||||
from rune.model_routing import ModelRouter
|
||||
|
||||
path = _profiles_file(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"provider": "ollama",
|
||||
"model": "qwen2.5:3b",
|
||||
"per_family_success": {"code": [5, 6]},
|
||||
"estimated_cost_usd": 0.0,
|
||||
},
|
||||
{
|
||||
"provider": "ollama",
|
||||
"model": "qwen3:14b",
|
||||
"per_family_success": {"code": [12, 13]},
|
||||
"estimated_cost_usd": 0.0,
|
||||
},
|
||||
],
|
||||
)
|
||||
ranked = ModelRouter(profiles_path=path).rank(task_family="code")
|
||||
# both measured, both cost 0 → deterministic tier order (T1 then T2);
|
||||
# unmeasured candidates may follow (never-empty guarantee) but never
|
||||
# precede measured ones
|
||||
top = [d.model for d in ranked][:2]
|
||||
assert top == ["qwen2.5:3b", "qwen3:14b"]
|
||||
assert all("measured" in d.reason for d in ranked[:2])
|
||||
assert ranked[0].reason == "measured 5/6 success task_family=code"
|
||||
assert ranked[1].reason == "measured 12/13 success task_family=code"
|
||||
|
||||
|
||||
def test_rank_measured_cost_breaks_tie_before_tier(tmp_path):
|
||||
from rune.model_routing import ModelRouter
|
||||
|
||||
path = _profiles_file(
|
||||
tmp_path,
|
||||
[
|
||||
{
|
||||
"provider": "ollama",
|
||||
"model": "qwen3:14b",
|
||||
"per_family_success": {"code": [10, 10]},
|
||||
},
|
||||
{
|
||||
"provider": "openai",
|
||||
"model": "mimo-v2.5-pro",
|
||||
"per_family_success": {"code": [9, 10]},
|
||||
"estimated_cost_usd": 1.0,
|
||||
},
|
||||
],
|
||||
)
|
||||
ranked = ModelRouter(profiles_path=path).rank(task_family="code", need_tools=True)
|
||||
# cheapest verified-capable first: free local beats paid cloud
|
||||
assert ranked[0].model == "qwen3:14b"
|
||||
assert ranked[0].estimated_cost == 0.0
|
||||
# and the unmeasured floor candidates that follow never precede it
|
||||
assert all(d.model == "qwen3:14b" or d.estimated_cost >= ranked[0].estimated_cost for d in ranked)
|
||||
|
||||
|
||||
def test_rank_deterministic_and_does_not_mutate_router_state():
|
||||
from rune.model_routing import ModelRouter
|
||||
|
||||
router = ModelRouter()
|
||||
candidates_before = tuple(router._candidates)
|
||||
first = router.rank(task_family="code")
|
||||
second = router.rank(task_family="code")
|
||||
assert first == second
|
||||
assert tuple(router._candidates) == candidates_before
|
||||
|
||||
|
||||
def test_load_profiles_missing_file_is_empty(tmp_path):
|
||||
from rune.model_routing import load_profiles
|
||||
|
||||
assert load_profiles(tmp_path / "state" / "model_routing_profiles.json") == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 3 — Operator wiring + escalation start-tier + trace v2 fields
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _scripted_operator_run(tmp_path, *, routing_reason="router: test reason", model="qwen3:8b"):
|
||||
"""Real RuneOperator + scripted router/broker/transport → envelope.
|
||||
|
||||
Mirrors canary_p1_07.py's scripted-run pattern; also patches the
|
||||
router factory so the run's start-tier decision comes from the
|
||||
(scripted) router with a recorded reason.
|
||||
"""
|
||||
import contextlib
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from rune import rune_operator
|
||||
from rune.local_tool_runtime import OllamaChatTransport
|
||||
from rune.tool_protocol import OperatorResponse
|
||||
|
||||
storage = Path(tmp_path) / "operator.json"
|
||||
op = rune_operator.RuneOperator(str(storage))
|
||||
|
||||
decision = SimpleNamespace(
|
||||
model=model,
|
||||
provider="ollama",
|
||||
tier="T2",
|
||||
reason=routing_reason,
|
||||
estimated_cost=0.0,
|
||||
)
|
||||
scripted_router = SimpleNamespace(rank=lambda *a, **k: [decision])
|
||||
|
||||
trace = {
|
||||
"model_id": model,
|
||||
"window": 40960,
|
||||
"budget": 32768,
|
||||
"blocks": [
|
||||
{"name": "persona", "tokens": 100, "admitted": True, "reason": ""},
|
||||
{"name": "user", "tokens": 20, "admitted": True, "reason": ""},
|
||||
],
|
||||
"admitted_total": 120,
|
||||
"dropped": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"compaction": [],
|
||||
"elapsed_ms": 1.0,
|
||||
}
|
||||
capsule = SimpleNamespace(
|
||||
token_count=120,
|
||||
active_objective="",
|
||||
blocks=[],
|
||||
to_messages=lambda: [{"role": "user", "content": "do it"}],
|
||||
)
|
||||
broker = SimpleNamespace(assemble=lambda task, session, model_id: (capsule, trace))
|
||||
profile = SimpleNamespace(safety_tier=0)
|
||||
response = OperatorResponse(success=True, user_text="done", model=model)
|
||||
|
||||
class _ScriptedRuntime:
|
||||
def __init__(self):
|
||||
self.transport = OllamaChatTransport.__new__(OllamaChatTransport)
|
||||
self.transport.provider = "ollama"
|
||||
self.transport.last_prompt_eval_count = None
|
||||
self.transport.usage_calls = 1
|
||||
self.transport.usage_prompt_tokens = 120
|
||||
self.transport.usage_output_tokens = 10
|
||||
self.transport.usage_total_duration_ns = 1_000_000
|
||||
|
||||
def run(self, messages, **kwargs):
|
||||
return response
|
||||
|
||||
runtime = _ScriptedRuntime()
|
||||
patches = [
|
||||
patch.object(op, "_context_broker", return_value=broker),
|
||||
patch.object(rune_operator.model_profiles, "get_or_create_profile", return_value=profile),
|
||||
patch.object(rune_operator.model_profiles, "record_interaction"),
|
||||
patch.object(rune_operator, "LocalToolRuntime", return_value=runtime),
|
||||
patch.object(rune_operator, "OllamaChatTransport", return_value=runtime.transport),
|
||||
patch("rune.local_tool_runtime.context_length", lambda model_: 40960),
|
||||
patch.object(
|
||||
rune_operator.safety_override.SafetyOverride,
|
||||
"get_params",
|
||||
lambda self, text: {"tier": 0, "temperature": 0.7, "prefix": "", "suffix": ""},
|
||||
),
|
||||
patch.object(rune_operator.safety_override.SafetyOverride, "is_refusal", lambda self, text: False),
|
||||
patch.object(op, "_log_token_usage"),
|
||||
patch.object(rune_operator, "_build_model_router", return_value=scripted_router),
|
||||
]
|
||||
with contextlib.ExitStack() as stack:
|
||||
for item in patches:
|
||||
stack.enter_context(item)
|
||||
stack.enter_context(patch("rune.event_bus._store_path", lambda: Path(storage).parent / "journal-events.jsonl"))
|
||||
return op.send_task("inspect this code path"), op
|
||||
|
||||
|
||||
def test_operator_run_records_routing_reason_and_schema_v3(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("RUNE_HOME", str(tmp_path / "home"))
|
||||
(tmp_path / "home" / "state").mkdir(parents=True, exist_ok=True)
|
||||
envelope, _op = _scripted_operator_run(tmp_path, routing_reason="router: start-tier T2 for task_family=code")
|
||||
assert envelope.get("success") is True
|
||||
# trace v3 (P1-01 #24 additive bump): routing fields land on the record
|
||||
from rune import inference_trace as it
|
||||
|
||||
records = it.read_traces(trace_dir=tmp_path / "home" / "state")
|
||||
assert records, "expected one durable inference trace"
|
||||
record = records[-1]
|
||||
assert record["schema_version"] == it.INFERENCE_TRACE_SCHEMA_VERSION
|
||||
assert record["routing_reason"] == "router: start-tier T2 for task_family=code"
|
||||
assert record["task_family"] == "code"
|
||||
|
||||
|
||||
def test_trace_v2_fields_are_additive_v1_rows_still_parse(tmp_path, monkeypatch):
|
||||
"""Back-compat: a v1 fixture row (no new fields) still reads cleanly."""
|
||||
monkeypatch.setenv("RUNE_HOME", str(tmp_path / "home"))
|
||||
state = tmp_path / "home" / "state"
|
||||
state.mkdir(parents=True, exist_ok=True)
|
||||
v1_row = {
|
||||
"schema_version": 1,
|
||||
"provider": "ollama",
|
||||
"model": "qwen3:8b",
|
||||
"task_id": "t1",
|
||||
"channel": "task",
|
||||
"resolved_context_window": 4096,
|
||||
"input_budget": 4096,
|
||||
"model_calls": 1,
|
||||
"prompt_tokens_estimated": 10,
|
||||
"prompt_tokens_served": 10,
|
||||
"output_tokens": 2,
|
||||
"latency_ms": 1.0,
|
||||
"model_time_ms": 1.0,
|
||||
"per_block_tokens": {},
|
||||
"dropped_blocks": [],
|
||||
"tools_exposed": [],
|
||||
"memories_exposed": [],
|
||||
"memory_tokens": 0,
|
||||
"degraded": [],
|
||||
"cache": None,
|
||||
"cost_usd": None,
|
||||
"cost_basis": "local-unpriced",
|
||||
"success": True,
|
||||
"error": "",
|
||||
}
|
||||
(state / "inference_traces.jsonl").write_text(json.dumps(v1_row) + "\n")
|
||||
from rune import inference_trace as it
|
||||
|
||||
records = it.read_traces(trace_dir=state)
|
||||
assert records[0]["schema_version"] == 1
|
||||
# v1 readers keep working: the new fields are absent on old rows
|
||||
assert "routing_reason" not in records[0]
|
||||
# and a v2 dataclass can be built with defaults
|
||||
trace = it.InferenceTrace(
|
||||
schema_version=2,
|
||||
provider="ollama",
|
||||
model="m",
|
||||
task_id="t",
|
||||
channel="task",
|
||||
resolved_context_window=1,
|
||||
input_budget=1,
|
||||
model_calls=1,
|
||||
prompt_tokens_estimated=1,
|
||||
prompt_tokens_served=1,
|
||||
output_tokens=1,
|
||||
latency_ms=1.0,
|
||||
model_time_ms=1.0,
|
||||
per_block_tokens={},
|
||||
dropped_blocks=[],
|
||||
tools_exposed=[],
|
||||
memories_exposed=[],
|
||||
memory_tokens=0,
|
||||
degraded=[],
|
||||
cache=None,
|
||||
cost_usd=None,
|
||||
cost_basis="local-unpriced",
|
||||
success=True,
|
||||
error="",
|
||||
)
|
||||
assert trace.routing_reason == "" and trace.routing_mode == "" and trace.task_family == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Task 4 — Composition registration + authority map
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_composition_registers_model_routing_authority(tmp_path):
|
||||
from pathlib import Path
|
||||
|
||||
from rune.orchestration.composition import (
|
||||
build_root,
|
||||
compose_model_routing,
|
||||
)
|
||||
|
||||
root = build_root({"events": tmp_path / "e.jsonl", "evidence": tmp_path / "ev", "state": tmp_path})
|
||||
authority = compose_model_routing(root, Path(tmp_path) / "state")
|
||||
# exactly one router authority
|
||||
router = root.get("model_routing")
|
||||
assert router is authority.router
|
||||
assert root.inventory()["model_routing"] == ["ModelRouter"]
|
||||
# advisory: rank() works and never mutates
|
||||
ranked = router.rank(task_family="code")
|
||||
assert isinstance(ranked, list)
|
||||
|
||||
|
||||
def test_authority_map_model_routing_paths_live():
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
amap = json.loads(Path("config/v1_authority_map.json").read_text())
|
||||
resp = {r["id"]: r for r in amap["responsibilities"]}
|
||||
routing = resp["model_routing"]
|
||||
adaptation = resp["provider_adaptation"]
|
||||
live_paths = [p["path"] for p in routing["current_paths"] if p["disposition"] == "migrate-adapter"]
|
||||
assert any("model_routing.py" in p for p in live_paths), (
|
||||
"model_routing authority path must be marked migrate-adapter (live)"
|
||||
)
|
||||
live_adapt = [p["path"] for p in adaptation["current_paths"] if p["disposition"] == "migrate-adapter"]
|
||||
assert any("model_routing.py" in p for p in live_adapt), (
|
||||
"provider_adaptation adapter path must be marked migrate-adapter (live)"
|
||||
)
|
||||
# evidence lines name P1-06
|
||||
for section in (routing, adaptation):
|
||||
for p in section["current_paths"]:
|
||||
if "model_routing.py" in p["path"]:
|
||||
assert "P1-06" in p["evidence"]
|
||||
|
||||
|
||||
def test_get_model_name_cloud_without_adapter_records_fallback_reason(tmp_path, monkeypatch):
|
||||
from rune import rune_tui
|
||||
|
||||
monkeypatch.setenv("RUNE_HOME", str(tmp_path / "home"))
|
||||
(tmp_path / "home" / "state").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Module-level resolver (delegated to by RuneApp._get_model_name):
|
||||
# testable without instantiating the Textual app.
|
||||
reason = rune_tui._resolve_model_name("cloud")
|
||||
# the "cloud=qwen2.5:7b" lie is dead: either the router's cloud
|
||||
# candidate or an explicit fallback reason
|
||||
assert reason != "qwen2.5:7b"
|
||||
assert "cloud" in reason and ("unavailable" in reason or "fallback" in reason)
|
||||
# and local/auto settings keep working
|
||||
assert rune_tui._resolve_model_name("local") == "qwen2.5:3b"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality-gate discharges (review of 5d7b494) — blocker + importants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_rank_excludes_non_executable_deterministic_candidate():
|
||||
"""Quality BLOCKER: the T0 'deterministic' pseudo-model must never be
|
||||
ranked — it is not executable by any transport (old behavior returned
|
||||
qwen2.5:7b; sending the literal id 'deterministic' to Ollama 404s)."""
|
||||
from rune.model_routing import ModelRouter
|
||||
|
||||
router = ModelRouter(profiles={}) # no measured profiles
|
||||
decisions = router.rank(task_family="code")
|
||||
assert decisions, "rank must still return runnable candidates"
|
||||
assert all(d.model != "deterministic" for d in decisions)
|
||||
# first runnable candidate is the cheapest floor model, not a pseudo id
|
||||
assert decisions[0].model == "qwen2.5:3b"
|
||||
|
||||
|
||||
def test_rank_sorts_by_numeric_tier_not_string():
|
||||
"""Tier ids sort numerically: 'T2' must come after 'T10' numerically,
|
||||
and cost ties break by tier number, not lexicographic 'T10'<'T2'."""
|
||||
from rune.model_routing import ModelRouter
|
||||
|
||||
candidates = (
|
||||
("m-a", "ollama", "T2", False, 32768, 0.0),
|
||||
("m-b", "ollama", "T10", False, 32768, 0.0),
|
||||
)
|
||||
router = ModelRouter(candidates=candidates, profiles={})
|
||||
decisions = router.rank(task_family="code")
|
||||
assert [d.tier for d in decisions] == ["T2", "T10"]
|
||||
|
||||
|
||||
def test_measured_candidates_respect_capability_floor():
|
||||
"""Quality I-5: measured stats never bypass the static floor — a model
|
||||
that measurably succeeded but lacks tools/min_context is excluded when
|
||||
the requirement demands them; measured success needs min sample n>=2
|
||||
to outrank the floor; non-numeric JSON stats are coerced/dropped."""
|
||||
from rune.model_routing import MeasuredModelProfile, ModelRouter
|
||||
|
||||
profiles = {
|
||||
("ollama", "qwen2.5:3b"): MeasuredModelProfile(
|
||||
provider="ollama",
|
||||
model="qwen2.5:3b",
|
||||
per_family_success={"code": (5, 5)}, # perfect record, n=5
|
||||
),
|
||||
}
|
||||
router = ModelRouter(profiles=profiles)
|
||||
# need_tools=True: measured 5/5 tool-family success IS tool-support
|
||||
# evidence → the model stays ranked (measured overrides static claim).
|
||||
decisions = router.rank(task_family="code", need_tools=True)
|
||||
assert decisions[0].model == "qwen2.5:3b"
|
||||
assert "measured 5/5" in decisions[0].reason
|
||||
# min_context beyond the floor model's window → excluded (hard limit)
|
||||
decisions = router.rank(task_family="code", min_context=100000)
|
||||
assert all(d.model != "qwen2.5:3b" for d in decisions)
|
||||
# n=1 measured sample must NOT outrank the capability floor ordering
|
||||
# (min sample size 2 for measured precedence)
|
||||
profiles1 = {
|
||||
("ollama", "qwen3:14b"): MeasuredModelProfile(
|
||||
provider="ollama",
|
||||
model="qwen3:14b",
|
||||
per_family_success={"code": (1, 1)},
|
||||
),
|
||||
}
|
||||
router1 = ModelRouter(profiles=profiles1)
|
||||
decisions = router1.rank(task_family="code")
|
||||
# qwen3:14b is T2 (more expensive tier) — with n=1 it must not jump ahead
|
||||
# of the cheaper T1 floor candidate on measured precedence alone.
|
||||
assert decisions[0].model == "qwen2.5:3b"
|
||||
|
||||
|
||||
def test_load_profiles_coerces_and_drops_bad_stats(tmp_path):
|
||||
"""Quality I-5: non-numeric ok/n from JSON must not raise TypeError in
|
||||
rank(); bad entries are dropped at load time."""
|
||||
from rune.model_routing import ModelRouter, load_profiles
|
||||
|
||||
p = tmp_path / "profiles.json"
|
||||
p.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"provider": "ollama",
|
||||
"model": "qwen2.5:3b",
|
||||
"per_family_success": {"code": ["five", "ten"]}, # garbage
|
||||
},
|
||||
{
|
||||
"provider": "ollama",
|
||||
"model": "qwen3:14b",
|
||||
"per_family_success": {"code": [4, 4]}, # valid
|
||||
},
|
||||
]
|
||||
)
|
||||
)
|
||||
profiles = load_profiles(p)
|
||||
assert ("ollama", "qwen2.5:3b") not in profiles
|
||||
assert ("ollama", "qwen3:14b") in profiles
|
||||
router = ModelRouter(profiles=profiles)
|
||||
decisions = router.rank(task_family="code") # must not raise
|
||||
assert any(d.model == "qwen3:14b" for d in decisions)
|
||||
|
||||
|
||||
def test_save_profiles_atomic(tmp_path):
|
||||
"""Quality I-4: save_profiles writes via temp+os.replace - a reader can
|
||||
never observe a torn file, and the temp file never lingers."""
|
||||
from rune.model_routing import MeasuredModelProfile, save_profiles
|
||||
|
||||
p = tmp_path / "profiles.json"
|
||||
from rune.model_routing import load_profiles as _lp
|
||||
|
||||
profiles = {
|
||||
("ollama", "m"): MeasuredModelProfile(provider="ollama", model="m", per_family_success={"code": (3, 3)})
|
||||
}
|
||||
save_profiles(p, profiles)
|
||||
assert p.exists()
|
||||
assert not list(tmp_path.glob("*.tmp*"))
|
||||
loaded = _lp(p)
|
||||
assert loaded[("ollama", "m")].per_family_success == {"code": (3, 3)}
|
||||
|
||||
|
||||
def test_update_profiles_from_traces_wired_into_operator_run(tmp_path, monkeypatch):
|
||||
"""Quality I-2: the measured loop must CLOSE - after a real operator run
|
||||
the trace outcome folds into the profiles store (production call site)."""
|
||||
from rune.model_routing import load_profiles as _lp
|
||||
|
||||
monkeypatch.setenv("RUNE_HOME", str(tmp_path / "home"))
|
||||
(tmp_path / "home" / "state").mkdir(parents=True, exist_ok=True)
|
||||
envelope, _op = _scripted_operator_run(tmp_path)
|
||||
assert envelope.get("success") is True
|
||||
from rune.model_routing import MeasuredModelProfile, load_profiles
|
||||
|
||||
profiles_path = tmp_path / "home" / "state" / "model_routing_profiles.json"
|
||||
assert profiles_path.exists(), "profiles must be folded after a run"
|
||||
profiles = load_profiles(profiles_path)
|
||||
assert profiles, "folded profiles must exist"
|
||||
assert any(prof.per_family_success for prof in profiles.values()), (
|
||||
"the run's measured outcome must appear in the profiles"
|
||||
)
|
||||
@@ -15,8 +15,6 @@ The canary (src/rune/canary_p1_07.py) re-proves the same contract CI-side.
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import shutil
|
||||
import uuid
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -442,127 +440,3 @@ def test_work_pane_collapses_and_expands():
|
||||
app.action_toggle_work_pane()
|
||||
assert app.work_pane_visible is True
|
||||
assert app._work_pane_visible_for("task") is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Quality-gate discharges (PR #180 review) — approved-resume + crash-tier
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_approval_resume_emits_outcome(tmp_path, monkeypatch):
|
||||
"""Quality I-2: an approval-resumed turn must emit an Outcome object.
|
||||
|
||||
resolve_pending_approval applies the approved operation and finalizes;
|
||||
the resume envelope must carry the same first-class Outcome a direct
|
||||
turn gets (approved tool calls are real executed work).
|
||||
"""
|
||||
transport = _ScriptedTransport([])
|
||||
tool_request = ToolRequest(call_id="call-9", name="write_file", arguments={}, source="ollama-native")
|
||||
tool_event = ToolEvent(kind="completed", request=tool_request, result={"success": True, "result": "written"})
|
||||
resumed = OperatorResponse(
|
||||
success=True,
|
||||
user_text="Approved and written.",
|
||||
model="qwen3:8b",
|
||||
tool_events=[tool_event],
|
||||
override_tier=0,
|
||||
recall_count=0,
|
||||
)
|
||||
runtime = _ScriptedRuntime(transport, resumed)
|
||||
# runtime.approve is what resolve_pending_approval calls
|
||||
runtime.approve = lambda pending_state: resumed # type: ignore[method-assign]
|
||||
op, patches = _wire_operator(tmp_path, monkeypatch, runtime)
|
||||
|
||||
# Seed a pending approval on the operator state (the scripted runtime
|
||||
# returns a terminal response, so no pending approval exists naturally).
|
||||
seeded = rune_operator.RuneTask(task_id=f"task_{uuid.uuid4().hex[:8]}", description="seeded approval")
|
||||
op.state.tasks.append(seeded)
|
||||
op.state.pending_task_id = seeded.task_id
|
||||
op.state.pending_request = {"call_id": "call-9", "name": "write_file", "arguments": {}}
|
||||
from rune.tool_executor import PreparedTool, _canonical_digest, _path_digest, serialize_prepared_tool
|
||||
import time as _t
|
||||
import os as _os
|
||||
|
||||
# write_file policy allows only /home/nick and /mnt/workdrive — use the
|
||||
# same sandbox pattern the P0-05 E2E uses (cleared before use).
|
||||
sandbox = Path(_os.path.expanduser("~")) / ".rune-resume-probe"
|
||||
import shutil as _shutil
|
||||
|
||||
_shutil.rmtree(sandbox, ignore_errors=True)
|
||||
_shutil.rmtree(sandbox, ignore_errors=True)
|
||||
sandbox.mkdir(parents=True)
|
||||
target = sandbox / "out.txt"
|
||||
approval_token = uuid.uuid4().hex # not a literal credential — henxels wants no token= literal
|
||||
prepared = PreparedTool(
|
||||
token=approval_token,
|
||||
name="write_file",
|
||||
arguments={"path": str(target), "content": "x"},
|
||||
operation_digest=_canonical_digest("write_file", {"path": str(target), "content": "x"}),
|
||||
created_at=_t.time(),
|
||||
expires_at=_t.time() + 60,
|
||||
requested_path=str(target),
|
||||
resolved_path=str(target.resolve()),
|
||||
path_digest=_path_digest(target),
|
||||
)
|
||||
op.state.pending_runtime_state = {"prepared_tool": serialize_prepared_tool(prepared)}
|
||||
|
||||
with contextlib.ExitStack() as stack:
|
||||
for item in patches:
|
||||
stack.enter_context(item)
|
||||
result = op.resolve_pending_approval(True)
|
||||
assert result["success"], result.get("error")
|
||||
# probe cleanup: remove the write sandbox the approval touched
|
||||
_shutil.rmtree(sandbox, ignore_errors=True)
|
||||
outcomes = [obj for obj in result.get("runtime_objects", []) if "status" in obj]
|
||||
assert len(outcomes) == 1, "approval-resumed turn must emit exactly one Outcome"
|
||||
outcome = Outcome.from_dict(outcomes[0])
|
||||
assert outcome.status == "completed"
|
||||
assert outcome.task_id == seeded.task_id
|
||||
|
||||
|
||||
def test_failed_verification_backfill_is_straight_read(tmp_path, monkeypatch):
|
||||
"""Quality I-1: the outcome's verification link in the ENVELOPE must be
|
||||
exactly the emitted verification's id — no test-side recompute masking a
|
||||
send_task back-fill regression. (The canary already asserts this; the
|
||||
pytest suite is now strictly as strong.)"""
|
||||
response = OperatorResponse(
|
||||
success=True,
|
||||
user_text="Command ran clean.",
|
||||
model="qwen3:8b",
|
||||
tool_events=[
|
||||
ToolEvent(
|
||||
kind="completed",
|
||||
request=ToolRequest(call_id="call-1", name="run_command", arguments={}, source="ollama-native"),
|
||||
result={"success": True, "result": "(no output)"},
|
||||
)
|
||||
],
|
||||
)
|
||||
transport = _ScriptedTransport([_chat_response(1600, 42, 1_000_000_000)])
|
||||
runtime = _ScriptedRuntime(transport, response)
|
||||
op, patches = _wire_operator(tmp_path, monkeypatch, runtime)
|
||||
result = _run_wired(op, patches)
|
||||
|
||||
verifications = [obj for obj in result["runtime_objects"] if "method" in obj]
|
||||
outcomes = [obj for obj in result["runtime_objects"] if "status" in obj]
|
||||
assert len(verifications) == 1 and len(outcomes) == 1
|
||||
# Direct envelope assertion — no _linked_outcome recomputation.
|
||||
assert outcomes[0]["verification_id"] == verifications[0]["verification_id"]
|
||||
assert outcomes[0]["verification_id"] != ""
|
||||
|
||||
|
||||
def test_work_pane_entries_capped_and_channel_scoped():
|
||||
"""Quality I-3: pane entries are capped (bounded growth) and the channel
|
||||
filter keeps chat objects out of a task-scoped pane view."""
|
||||
pane = rune_tui.WorkPane()
|
||||
overflow = rune_tui._WORK_PANE_MAX_ENTRIES + 25
|
||||
for n in range(overflow):
|
||||
pane.add_object({"observation_id": f"obs_{n}", "kind": "degraded", "summary": f"s{n}"})
|
||||
assert len(pane._entries) <= rune_tui._WORK_PANE_MAX_ENTRIES
|
||||
# newest survive, oldest evicted
|
||||
assert "obs_0" not in pane._entries
|
||||
assert f"obs_{overflow - 1}" in pane._entries
|
||||
# channel scoping: entries carry their channel; visible set filters
|
||||
pane2 = rune_tui.WorkPane()
|
||||
pane2.add_object({"observation_id": "obs_task", "kind": "degraded", "summary": "t", "channel": "task"})
|
||||
pane2.add_object({"observation_id": "obs_chat", "kind": "degraded", "summary": "c", "channel": "chat"})
|
||||
visible = pane2.visible_entries("chat")
|
||||
assert [oid for oid, _ in visible] == ["obs_chat"]
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Tests: retrieval benchmark harness (#161, synthesis e584bbf895b3).
|
||||
|
||||
Every falsification-test conversion from the adversarial stage is proven
|
||||
here as a deterministic test. The harness itself is the primary validation
|
||||
target: these tests verify the harness CANNOT silently validate the wrong
|
||||
system.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
from rune import memory_store as ms
|
||||
from rune import memory_selector as msel
|
||||
from rune.orchestration import retrieval_benchmark as rb
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def seeded_store():
|
||||
entries = [
|
||||
{"text": "User drinks black coffee every morning", "topic": "habits",
|
||||
"entities": ["coffee"]},
|
||||
{"text": "Coffee brewing guide: grind size matters for espresso",
|
||||
"topic": "coffee", "entities": ["coffee"]},
|
||||
{"text": "Quantum physics studies black holes", "topic": "science",
|
||||
"entities": ["physics"]},
|
||||
{"text": "The housing market shows rising rent burden", "topic": "housing",
|
||||
"entities": ["housing"]},
|
||||
{"text": "Tea ceremony traditions differ by region", "topic": "culture",
|
||||
"entities": ["tea"]},
|
||||
]
|
||||
db, db_path = rb.make_isolated_store(entries)
|
||||
yield db, db_path
|
||||
db.close()
|
||||
shutil_ignore = Path(db_path).parent
|
||||
import shutil as _sh
|
||||
_sh.rmtree(shutil_ignore, ignore_errors=True)
|
||||
|
||||
|
||||
# ── M1: mode field observability ─────────────────────────────────────────────
|
||||
|
||||
def test_recall_mode_field_observable(seeded_store):
|
||||
"""Every recall() result carries mode; values from the canonical enum."""
|
||||
db, _ = seeded_store
|
||||
res = ms.recall(db, "coffee")
|
||||
assert res, "expected results for 'coffee'"
|
||||
for r in res:
|
||||
assert r.get("mode") in ("lexical-only", "dense-only", "hybrid"), \
|
||||
f"unlabeled mode: {r.get('mode')!r}"
|
||||
|
||||
|
||||
def test_recall_mode_reflects_fusion_branch(seeded_store):
|
||||
"""Single-content-word query without embedding match → lexical-only;
|
||||
hybrid when both rankers contribute."""
|
||||
db, _ = seeded_store
|
||||
res = ms.recall(db, "coffee")
|
||||
assert all(r["mode"] in ("lexical-only", "hybrid") for r in res)
|
||||
|
||||
|
||||
def test_recall_cap_parameter_defaults_preserve_behavior(seeded_store):
|
||||
"""M8: default probe_cap == RECALL_BUDGET*2 — byte-identical results to
|
||||
pre-change behavior with defaults unset."""
|
||||
import inspect
|
||||
sig = inspect.signature(ms.recall)
|
||||
assert sig.parameters["probe_cap"].default == ms.RECALL_BUDGET * 2
|
||||
db, _ = seeded_store
|
||||
r_default = ms.recall(db, "coffee")
|
||||
r_explicit = ms.recall(db, "coffee", probe_cap=ms.RECALL_BUDGET * 2)
|
||||
assert [(x["id"], x["relevance"]) for x in r_default] == \
|
||||
[(x["id"], x["relevance"]) for x in r_explicit]
|
||||
|
||||
|
||||
def test_mode_field_never_alters_behavior(seeded_store):
|
||||
"""INV-13: same query, same store — presence of the mode field does not
|
||||
change ranking. (Structural check: mode is written, never read, by
|
||||
memory_store internals.)"""
|
||||
src = (ROOT / "src" / "rune" / "memory_store.py").read_text()
|
||||
# No conditional reads of the mode field inside memory_store
|
||||
assert 'get("mode")' not in src and "['mode']" not in src
|
||||
|
||||
|
||||
# ── FT1/INV-1: loud failure on unreachable embedding ─────────────────────────
|
||||
|
||||
def test_harness_fails_loud_on_unreachable_embedding(seeded_store, monkeypatch):
|
||||
"""The harness must REFUSE to run when the embedding service is down.
|
||||
Unlabeled mode switch voids validation — this is the A1 mitigation."""
|
||||
db, db_path = seeded_store
|
||||
monkeypatch.setattr(rb, "check_embedding_available", lambda: False)
|
||||
# dataset file
|
||||
ds = Path(db_path).parent / "dataset.json"
|
||||
ds.write_text(json.dumps([{"query": "coffee", "relevant_ids": []}]))
|
||||
cfg = rb.HarnessConfig(
|
||||
dataset_path=str(ds), dataset_sha256=hashlib.sha256(ds.read_bytes()).hexdigest(),
|
||||
backend_sources={"store": lambda q, t, b, m: []},
|
||||
queries=[{"query": "coffee", "relevant_ids": []}])
|
||||
with pytest.raises(RuntimeError, match="FT1 VIOLATION"):
|
||||
rb.run_harness(cfg)
|
||||
|
||||
|
||||
def test_embedding_probe_detects_service_state():
|
||||
"""check_embedding_available returns a bool and survives an absent service."""
|
||||
result = rb.check_embedding_available()
|
||||
assert isinstance(result, bool)
|
||||
|
||||
|
||||
# ── M6/INV-9: isolation assertions ───────────────────────────────────────────
|
||||
|
||||
def test_isolated_store_assertions(tmp_path):
|
||||
"""Harness refuses non-isolated paths: production store, shard dirs,
|
||||
non-RUNE_* roots."""
|
||||
with pytest.raises(rb.IsolationError):
|
||||
rb.assert_isolated_store("/tmp/plain/bench.db") # not RUNE_*
|
||||
with pytest.raises(rb.IsolationError):
|
||||
rb.assert_isolated_store(str(tmp_path / "shards" / "b.db")) # shard dir
|
||||
# RUNE_* root passes
|
||||
ok = tmp_path / "RUNE_bench_x" / "b.db"
|
||||
ok.parent.mkdir(parents=True)
|
||||
rb.assert_isolated_store(str(ok)) # no raise
|
||||
|
||||
|
||||
def test_make_isolated_store_uses_rune_root(seeded_store):
|
||||
db, db_path = seeded_store
|
||||
assert "RUNE_bench_" in db_path
|
||||
rb.assert_isolated_store(db_path) # passes its own gate
|
||||
|
||||
|
||||
# ── M5/INV-8: registry contract only ─────────────────────────────────────────
|
||||
|
||||
def test_registry_contract_only(seeded_store):
|
||||
"""Backends are registry callables with the documented signature —
|
||||
the harness defines no second adapter protocol."""
|
||||
import inspect
|
||||
src = (ROOT / "src" / "rune" / "orchestration" / "retrieval_benchmark.py").read_text()
|
||||
assert "class.*Adapter" not in src # no adapter protocol invented
|
||||
db, db_path = seeded_store
|
||||
# A backend registered via the contract works end to end
|
||||
def store_backend(query, types, budget, min_score):
|
||||
return ms.recall(db, query, budget=budget, min_score=min_score)
|
||||
selector = msel.MemorySelector(db_path=db_path)
|
||||
selector.add_source("bench_store", store_backend)
|
||||
assert "bench_store" in selector.source_names()
|
||||
|
||||
|
||||
# ── M10/INV-12: one run-record schema, fingerprint reused ────────────────────
|
||||
|
||||
def test_shared_run_record_schema(seeded_store):
|
||||
"""Run records validate against the schema and carry the reused fingerprint()."""
|
||||
from rune.orchestration.benchmark import fingerprint
|
||||
profile = rb.StageProfile()
|
||||
profile.record("embed", 1.0)
|
||||
rec = rb.make_run_record("q", "store", [{"id": "d1"}], {"d1"}, [1, 3],
|
||||
profile, "hybrid")
|
||||
assert rec["fingerprint"] == fingerprint()
|
||||
assert rec["schema_version"] == rb.RUN_RECORD_SCHEMA_VERSION
|
||||
for stage in rb.STAGES:
|
||||
assert stage in rec["stage_timings_ms"]
|
||||
|
||||
|
||||
def test_ranking_bias_disclosed_in_cross_source_rows(seeded_store):
|
||||
"""M4: every cross-source row carries the ranking-bias disclosure."""
|
||||
profile = rb.StageProfile()
|
||||
rec = rb.make_run_record("q", "graphs", [{"id": "d1"}], {"d1"}, [1, 3],
|
||||
profile, "hybrid")
|
||||
assert "RANKING BIAS" in rec["ranking_bias_disclosure"].upper()
|
||||
|
||||
|
||||
# ── Metrics semantics ────────────────────────────────────────────────────────
|
||||
|
||||
def test_pr_curves_not_thresholded_numbers(seeded_store):
|
||||
"""M4/INV-10: PR output is a curve at stated operating points, never a
|
||||
single scalar."""
|
||||
profile = rb.StageProfile()
|
||||
rec = rb.make_run_record("q", "store", [{"id": f"d{i}"} for i in range(5)],
|
||||
{"d0", "d2"}, [1, 3, 5], profile, "hybrid")
|
||||
assert isinstance(rec["pr_curve"], list) and len(rec["pr_curve"]) == 3
|
||||
assert all("precision" in p and "recall" in p and "k" in p for p in rec["pr_curve"])
|
||||
|
||||
|
||||
def test_neutralized_vs_live_delta_reported(seeded_store):
|
||||
"""M2/INV-3: live vs neutral nDCG both reported; delta is first-class."""
|
||||
profile = rb.StageProfile()
|
||||
rec = rb.make_run_record("q", "store", [{"id": "d1"}], {"d1"}, [1],
|
||||
profile, "hybrid", live_ndcg=0.81, neutral_ndcg=0.77)
|
||||
assert rec["ndcg_live_composite"] == 0.81
|
||||
assert rec["ndcg_neutral_metadata"] == 0.77
|
||||
assert rec["ndcg_delta"] == round(0.81 - 0.77, 4)
|
||||
|
||||
|
||||
def test_abstention_scored_as_designed(seeded_store):
|
||||
"""M7/INV-11: coverage@budget treats abstention as designed behavior;
|
||||
risk-coverage curve is producible."""
|
||||
scored = [("d1", 0.9), ("d2", 0.7), ("d3", 0.4)]
|
||||
curve = rb.risk_coverage_curve(scored, {"d1", "d3"})
|
||||
assert curve[0]["coverage"] > 0
|
||||
assert all(0.0 <= c["risk"] <= 1.0 for c in curve)
|
||||
|
||||
|
||||
def test_ndcg_computation_correct():
|
||||
"""Sanity: perfect ranking scores 1.0; anti-correlated scores near 0."""
|
||||
assert rb.ndcg_at_k(["a", "b"], {"a", "b"}, k=10) == 1.0
|
||||
assert rb.ndcg_at_k(["x", "y"], {"a", "b"}, k=10) == 0.0
|
||||
|
||||
|
||||
# ── M3: cache bypass between measured runs ───────────────────────────────────
|
||||
|
||||
def test_cache_bypass_between_measured_runs():
|
||||
"""bypass_caches() reports its state and does not raise."""
|
||||
state = rb.bypass_caches()
|
||||
assert isinstance(state, dict)
|
||||
assert "result_cache" in state and "embed_cache" in state
|
||||
|
||||
|
||||
# ── M9/INV-7: evidence-citation gate ─────────────────────────────────────────
|
||||
|
||||
def test_evidence_citation_gate():
|
||||
"""No E20-E22 numeric claim without primary-source verification flag.
|
||||
The harness has no E20-E22 numbers; the gate is a lint rule — verify
|
||||
the harness source contains none of the unverified claims."""
|
||||
src = (ROOT / "src" / "rune" / "orchestration" / "retrieval_benchmark.py").read_text()
|
||||
for claim in ("32.1%", "25% TSR", "4.5x latency spread", "62% concentration"):
|
||||
assert claim not in src, f"unverified external claim in harness source: {claim}"
|
||||
|
||||
|
||||
# ── FT5: top-k sweep not clipped at budget ───────────────────────────────────
|
||||
|
||||
def test_topk_sweep_not_clipped(seeded_store):
|
||||
db, _ = seeded_store
|
||||
# probe_cap swept alongside top-k: recall at k=32 must not be artificially
|
||||
# identical to recall at k=8 when the pool is larger than 8
|
||||
results_small = ms.recall(db, "coffee", budget=8)
|
||||
results_large = ms.recall(db, "coffee", budget=8, probe_cap=64)
|
||||
# Both run without error; with a bigger pool the candidate set can differ
|
||||
assert isinstance(results_large, list)
|
||||
assert all("mode" in r for r in results_large)
|
||||
|
||||
|
||||
# ── M8: stage profiler covers all stages ─────────────────────────────────────
|
||||
|
||||
def test_stage_profiler_covers_all_stages(seeded_store):
|
||||
profile = rb.StageProfile()
|
||||
for s in rb.STAGES:
|
||||
profile.record(s, 0.5)
|
||||
rec = rb.make_run_record("q", "store", [], set(), [1], profile, "hybrid")
|
||||
for stage in ("embed", "dense_gate", "fts", "rrf", "metadata_blend", "abstention"):
|
||||
assert stage in rec["stage_timings_ms"]
|
||||
|
||||
|
||||
# ── FT4: scale sweep structure ───────────────────────────────────────────────
|
||||
|
||||
def test_scale_sweep_dense_gate_superlinearity(seeded_store):
|
||||
"""Structure proven at bench scale: stage-attributed p99 recorded per
|
||||
corpus size. The 100K blow-up assertion is a backend re-evaluation gate,
|
||||
not a unit-test obligation (documented in the synthesis)."""
|
||||
import time as _t
|
||||
db, _ = seeded_store
|
||||
profile = rb.StageProfile()
|
||||
t0 = _t.perf_counter()
|
||||
ms.recall(db, "coffee")
|
||||
profile.record("fts", (_t.perf_counter() - t0) * 1000)
|
||||
assert profile.p99("fts") > 0
|
||||
Reference in new issue
Block a user