Nick
df088fbe30
- GO CORE TURN ENGINE (src/go/rune-core): per-channel single-flight, channel-keyed context, K25 tail+snapshot store (D40 seam), live Ollama with the thinking split, PTY parity proven with Python stopped + the owner's 27B live with thinking surfaced - FULL-WIDTH SHARED BOX (D36): spans both panes, focus-signage placeholder, drafts survive focus round-trips, PTY-proven 110+140 - MODEL SPLIT (D37): pane-aware /model, per-pane record fields, status line; letter-stealing round 2 fixed (j/k type) - APPROVAL CARD (T5): gold y/n through /v1/approvals/resolve, stale revalidation, chained asks, PTY-proven both legs - PROVIDERS P1: 18 providers both languages, three-artifact drift gate, provider step in first-run (real names, no 'local' hardcode) - SCROLLBACK: /history depth persisted, ctrl+up/down + pgup/pgdn, dim ↑N indicator, deeper-page fetch, PTY-proven on real history - RETIREMENT rows 4+10 advanced: tool-executor dispatch wired (49/49+48/48), Go mode-gate door built, work-manager CLI parity
789 lines
32 KiB
Python
789 lines
32 KiB
Python
#!/usr/bin/env python3
|
|
"""gen_local_tool_runtime_truth.py — freeze truth for the local_tool_runtime
|
|
gap-close (dichotomy wave). Every branch of the NEW functions is exercised
|
|
against the LIVE Python oracle (src/rune/local_tool_runtime.py) with deep-
|
|
copied inputs; the frozen JSON is embedded verbatim in the Go truth battery
|
|
(src/go/local-tool-runtime/truth.json, go:embed).
|
|
|
|
Excluded from parity (documented, per auto_context/evidence_probe precedent):
|
|
- PreparedTool approve/reject fan-out: tokens are uuid4 and timestamps are
|
|
time.monotonic() — interpreter-nondeterministic. The plain needs_confirm
|
|
approval branch (executor without tool authority) IS frozen.
|
|
- register_window/reset_discovered_windows broker side effects: stubbed at
|
|
the seam (import succeeds, calls recorded, no state) — the Go side has no
|
|
broker; the seam boundary is the parity contract.
|
|
- _find_capability_executor on mutating tools: needs uuid tokens.
|
|
Frozen read-only execute_tool rows cover the routing contract.
|
|
|
|
Run: python3 scripts/gen_local_tool_runtime_truth.py
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import os
|
|
import sys
|
|
import types
|
|
import unittest.mock as mock
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
|
|
import rune.context_broker as context_broker
|
|
import rune.openai_compat_transport as openai_compat_transport
|
|
import rune.subsystem_health as subsystem_health
|
|
from rune import local_tool_runtime as ltr
|
|
from rune.local_tool_runtime import (
|
|
LocalToolRuntime,
|
|
OllamaChatTransport,
|
|
_extract_context_length,
|
|
)
|
|
|
|
|
|
import re
|
|
_UUID_CALL_ID = re.compile(r"^call_[0-9a-f]{32}$")
|
|
|
|
|
|
def wildcard_uuids(obj):
|
|
"""Replace nondeterministic _new_call_id() uuids with a wildcard —
|
|
Python text-fallback envelopes always mint fresh uuids; the id SHAPE
|
|
(call_<32 hex>) is the contract, the value is not."""
|
|
if isinstance(obj, dict):
|
|
return {
|
|
k: ("<new_call_id>" if (k in ("call_id", "tool_call_id") and isinstance(v, str) and _UUID_CALL_ID.match(v)) else wildcard_uuids(v))
|
|
for k, v in obj.items()
|
|
}
|
|
if isinstance(obj, list):
|
|
return [wildcard_uuids(x) for x in obj]
|
|
return obj
|
|
|
|
|
|
def freeze(obj):
|
|
"""JSON-safe projection: frozensets -> sorted lists, tuples -> lists."""
|
|
if isinstance(obj, frozenset) or isinstance(obj, set):
|
|
return sorted(obj)
|
|
if isinstance(obj, tuple):
|
|
return [freeze(x) for x in obj]
|
|
if isinstance(obj, dict):
|
|
return {str(k): freeze(v) for k, v in obj.items()}
|
|
if isinstance(obj, list):
|
|
return [freeze(x) for x in obj]
|
|
if isinstance(obj, BaseException):
|
|
return type(obj).__name__ + ": " + str(obj)
|
|
return obj
|
|
|
|
|
|
def resp_freeze(response):
|
|
d = asdict(response)
|
|
return freeze(d)
|
|
|
|
|
|
# ── transport fakes (deep-copied inputs, recorded calls) ─────────────────────
|
|
|
|
|
|
class FakeTransport:
|
|
def __init__(self, responses, *, native_tools=True, thinking=True):
|
|
self.responses = list(responses)
|
|
self.native_tools = native_tools
|
|
self.thinking = thinking
|
|
self.calls = []
|
|
|
|
def supports_tools(self, model):
|
|
return self.native_tools
|
|
|
|
def supports_thinking(self, model):
|
|
return self.thinking
|
|
|
|
def chat(self, *, model, messages, tools, options=None, think=None):
|
|
request = {
|
|
"model": model,
|
|
"messages": copy.deepcopy(messages),
|
|
"tools": copy.deepcopy(tools),
|
|
"options": copy.deepcopy(options),
|
|
"think": copy.deepcopy(think),
|
|
}
|
|
self.calls.append(request)
|
|
response = self.responses.pop(0)
|
|
if isinstance(response, BaseException):
|
|
raise response
|
|
if not isinstance(response, dict):
|
|
raise TypeError("fake response must be a message object")
|
|
return copy.deepcopy(response)
|
|
|
|
|
|
class ScriptExecutor:
|
|
def __init__(self, results):
|
|
self.results = list(results)
|
|
self.calls = []
|
|
|
|
def __call__(self, name, arguments):
|
|
self.calls.append((name, copy.deepcopy(arguments)))
|
|
if self.results:
|
|
return self.results.pop(0)
|
|
return {"success": True, "result": name}
|
|
|
|
|
|
# ── 1. _extract_context_length ────────────────────────────────────────────────
|
|
|
|
extract_rows = []
|
|
shown_cases = [
|
|
{"model_info": {"qwen2.context_length": 131072}},
|
|
{"model_info": {"qwen2.context_length": "131072", "other.context_length": 4096}},
|
|
{"model_info": {"qwen2.context_length": 0}},
|
|
{"model_info": "not a dict", "context_length": 8192},
|
|
{"context_length": 4096},
|
|
{"context_length": "4096"},
|
|
{"context_length": -5},
|
|
{},
|
|
{"model_info": {"arch.num_params": 7}},
|
|
{"model_info": {"qwen2.context_length": True}},
|
|
]
|
|
for shown in shown_cases:
|
|
want = _extract_context_length(copy.deepcopy(shown))
|
|
extract_rows.append({"shown": shown, "want": want})
|
|
|
|
# ── 2. _positive_int ─────────────────────────────────────────────────────────
|
|
|
|
positive_int_rows = []
|
|
for value in [5, 0, -3, True, False, "7", 3.7, None, [1], {"a": 1}, 10**12]:
|
|
positive_int_rows.append({"value": freeze(value), "want": OllamaChatTransport._positive_int(value)})
|
|
|
|
# ── 3. _model_capabilities / supports_tools / supports_thinking ──────────────
|
|
|
|
cap_rows = []
|
|
cap_cases = [
|
|
{"capabilities": ["completion", "tools", 5, None, "thinking"]},
|
|
{"capabilities": "tools"},
|
|
{},
|
|
{"capabilities": []},
|
|
]
|
|
for shown in cap_cases:
|
|
transport = OllamaChatTransport()
|
|
posts = []
|
|
|
|
def fake_post(path, payload, _shown=shown, _posts=posts):
|
|
_posts.append((path, copy.deepcopy(payload)))
|
|
return copy.deepcopy(_shown)
|
|
|
|
transport._post = fake_post
|
|
caps = sorted(transport._model_capabilities("m1"))
|
|
tools = transport.supports_tools("m1")
|
|
thinking = transport.supports_thinking("m1")
|
|
caps2 = sorted(transport._model_capabilities("m1"))
|
|
cap_rows.append({
|
|
"shown": shown,
|
|
"capabilities": caps,
|
|
"capabilities_second_call": caps2,
|
|
"supports_tools": tools,
|
|
"supports_thinking": thinking,
|
|
"posts": posts,
|
|
"context_lengths": freeze(transport._context_lengths),
|
|
})
|
|
|
|
# caching: two different models, two posts
|
|
transport = OllamaChatTransport()
|
|
posts = []
|
|
transport._post = lambda path, payload, _p=posts: (
|
|
_p.append((path, copy.deepcopy(payload))) or copy.deepcopy({"capabilities": ["tools"], "model_info": {"a.context_length": 2048}})
|
|
)
|
|
c1 = sorted(transport._model_capabilities("m1"))
|
|
c2 = sorted(transport._model_capabilities("m2"))
|
|
cap_rows.append({
|
|
"shown": "two-models",
|
|
"capabilities": c1 + c2,
|
|
"capabilities_second_call": None,
|
|
"supports_tools": None,
|
|
"supports_thinking": None,
|
|
"posts": posts,
|
|
"context_lengths": freeze(transport._context_lengths),
|
|
})
|
|
|
|
# ── 4. OllamaChatTransport.context_length ────────────────────────────────────
|
|
|
|
tctx_rows = []
|
|
for shown, scenario in [
|
|
({"model_info": {"qwen2.context_length": 4096}}, "reported"),
|
|
({}, "absent"),
|
|
]:
|
|
transport = OllamaChatTransport()
|
|
posts = []
|
|
transport._post = lambda path, payload, _s=shown, _p=posts: (
|
|
_p.append((path, copy.deepcopy(payload))) or copy.deepcopy(_s)
|
|
)
|
|
if scenario == "reported":
|
|
window = transport.context_length("m1")
|
|
window2 = transport.context_length("m1")
|
|
tctx_rows.append({
|
|
"scenario": scenario,
|
|
"shown": shown,
|
|
"window": window,
|
|
"window_second_call": window2,
|
|
"posts": posts,
|
|
})
|
|
else:
|
|
try:
|
|
transport.context_length("m1")
|
|
error = None
|
|
except ValueError as exc:
|
|
error = str(exc)
|
|
tctx_rows.append({"scenario": scenario, "shown": shown, "window": None, "error": error, "posts": posts})
|
|
|
|
# ── 5. module-level context_length ───────────────────────────────────────────
|
|
|
|
|
|
class _StubOllama:
|
|
instances = []
|
|
|
|
def __init__(self, base_url="http://localhost:11434", timeout=120):
|
|
self.base_url = base_url
|
|
self.instances.append(self)
|
|
|
|
def context_length(self, model):
|
|
return 4096
|
|
|
|
|
|
class _StubCloud:
|
|
def __init__(self):
|
|
pass
|
|
|
|
def context_length(self, model):
|
|
return 131072
|
|
|
|
|
|
def run_module_context_length(scenario, model, seed_cache):
|
|
ltr._CONTEXT_LENGTHS.clear()
|
|
ltr._CONTEXT_LENGTHS.update(copy.deepcopy(seed_cache))
|
|
failures = []
|
|
broker_resets = []
|
|
with mock.patch.object(ltr, "OllamaChatTransport", _StubOllama), \
|
|
mock.patch.object(openai_compat_transport, "is_cloud_pinned",
|
|
lambda m: scenario in ("cloud", "cloud_raises")), \
|
|
mock.patch.object(openai_compat_transport, "cloud_transport_for",
|
|
lambda m: _StubCloud()), \
|
|
mock.patch.object(subsystem_health, "record_failure",
|
|
lambda *a, **k: failures.append({"args": freeze(a), "kwargs": freeze(k)})), \
|
|
mock.patch.object(context_broker, "register_window",
|
|
lambda *a, **k: broker_resets.append(("register", freeze(a)))), \
|
|
mock.patch.object(context_broker, "reset_discovered_windows",
|
|
lambda: broker_resets.append(("reset", None))):
|
|
if scenario == "raises":
|
|
def boom(self, model):
|
|
raise RuntimeError("ollama down")
|
|
with mock.patch.object(_StubOllama, "context_length", boom):
|
|
window = ltr.context_length(model)
|
|
elif scenario == "cloud_raises":
|
|
class _BoomCloud:
|
|
def context_length(self, m):
|
|
raise RuntimeError("cloud down")
|
|
with mock.patch.object(openai_compat_transport, "cloud_transport_for", lambda m: _BoomCloud()):
|
|
window = ltr.context_length(model)
|
|
else:
|
|
window = ltr.context_length(model)
|
|
window2 = ltr.context_length(model)
|
|
return {
|
|
"scenario": scenario,
|
|
"model": model,
|
|
"seed_cache": freeze(seed_cache),
|
|
"window": window,
|
|
"window_second_call": window2,
|
|
"cache_after": freeze(ltr._CONTEXT_LENGTHS),
|
|
"failures": failures,
|
|
"broker": broker_resets,
|
|
"ollama_constructs": len(_StubOllama.instances),
|
|
}
|
|
|
|
|
|
_StubOllama.instances = []
|
|
module_ctx_rows = [
|
|
run_module_context_length("local", "qwen2.5:3b", {}),
|
|
run_module_context_length("cloud", "Verboo Code/glm-5.3-flash", {}),
|
|
run_module_context_length("cached", "m1", {"m1": 32768}),
|
|
run_module_context_length("raises", "m1", {}),
|
|
run_module_context_length("cloud_raises", "m1", {}),
|
|
]
|
|
_StubOllama.instances = []
|
|
|
|
# ── 6. reset_context_length_cache ────────────────────────────────────────────
|
|
|
|
broker_resets = []
|
|
ltr._CONTEXT_LENGTHS.clear()
|
|
ltr._CONTEXT_LENGTHS.update({"m1": 4096, "m2": 8192})
|
|
with mock.patch.object(context_broker, "reset_discovered_windows",
|
|
lambda: broker_resets.append("reset")):
|
|
ltr.reset_context_length_cache()
|
|
reset_row = {"cache_before": {"m1": 4096, "m2": 8192}, "cache_after": freeze(ltr._CONTEXT_LENGTHS),
|
|
"broker": broker_resets}
|
|
|
|
# ── 7. _deduplicate ──────────────────────────────────────────────────────────
|
|
|
|
from rune.tool_protocol import ToolRequest
|
|
|
|
dedup_rows = []
|
|
dedup_cases = [
|
|
[("1", "read_file", {"path": "/a", "limit": 5}), ("2", "read_file", {"limit": 5, "path": "/a"})],
|
|
[("1", "read_file", {"path": "/a"}), ("2", "read_file", {"path": "/b"})],
|
|
[("1", "list_directory", {}), ("2", "list_directory", {}), ("3", "grep_files", {"pattern": "x"})],
|
|
[],
|
|
]
|
|
for case in dedup_cases:
|
|
requests = [ToolRequest(cid, name, args, "ollama-native") for cid, name, args in case]
|
|
unique = LocalToolRuntime._deduplicate(copy.deepcopy(requests))
|
|
dedup_rows.append({
|
|
"requests": [{"call_id": r.call_id, "name": r.name, "arguments": r.arguments, "source": r.source} for r in requests],
|
|
"unique": [{"call_id": r.call_id, "name": r.name, "arguments": r.arguments, "source": r.source} for r in unique],
|
|
})
|
|
|
|
# ── 8. _tool_message ─────────────────────────────────────────────────────────
|
|
|
|
tool_message_rows = []
|
|
for request_args, result in [
|
|
({"path": "/tmp"}, {"success": True, "result": "listing", "z": 1, "a": [1, 2]}),
|
|
({"path": "/tmp"}, {"success": False, "error": "permission denied"}),
|
|
]:
|
|
request = ToolRequest("call-1", "list_directory", request_args, "ollama-native")
|
|
message = LocalToolRuntime._tool_message(request, copy.deepcopy(result))
|
|
tool_message_rows.append({
|
|
"request": {"call_id": request.call_id, "name": request.name, "arguments": request.arguments, "source": request.source},
|
|
"result": result,
|
|
"message": freeze(message),
|
|
})
|
|
|
|
# ── 9/10. request & event state roundtrips + errors ──────────────────────────
|
|
|
|
request_state_rows = []
|
|
event_state_rows = []
|
|
|
|
request = ToolRequest("call-9", "read_file", {"path": "/a"}, "text-fallback")
|
|
state = LocalToolRuntime._request_to_state(copy.deepcopy(request))
|
|
request_state_rows.append({"input": {"call_id": request.call_id, "name": request.name, "arguments": request.arguments, "source": request.source}, "state": freeze(state)})
|
|
|
|
for bad in ["nope", 5, None, [], {}, {"call_id": "", "name": "n", "arguments": {}, "source": "s"},
|
|
{"call_id": "c", "name": "", "arguments": {}, "source": "s"},
|
|
{"call_id": "c", "name": "n", "arguments": [], "source": "s"},
|
|
{"call_id": "c", "name": "n", "arguments": {}, "source": ""},
|
|
{"call_id": "c", "name": "n"}]:
|
|
try:
|
|
LocalToolRuntime._request_from_state(copy.deepcopy(bad))
|
|
error = None
|
|
except (TypeError, ValueError) as exc:
|
|
error = type(exc).__name__ + ": " + str(exc)
|
|
request_state_rows.append({"input": freeze(bad) if isinstance(bad, (dict, list)) else bad, "error": error})
|
|
|
|
event = ltr.ToolEvent("completed", request, {"success": True, "result": "ok"})
|
|
event_state_rows.append({"state": freeze(LocalToolRuntime._event_to_state(copy.deepcopy(event)))})
|
|
event_none = ltr.ToolEvent("error", request, None)
|
|
event_state_rows.append({"state": freeze(LocalToolRuntime._event_to_state(copy.deepcopy(event_none)))})
|
|
|
|
for bad in ["nope", 5, None, [], {}, {"kind": "", "request": state, "result": None},
|
|
{"kind": "completed", "request": "x", "result": None},
|
|
{"kind": "completed", "request": state, "result": "x"}]:
|
|
try:
|
|
LocalToolRuntime._event_from_state(copy.deepcopy(bad))
|
|
error = None
|
|
except (TypeError, ValueError) as exc:
|
|
error = type(exc).__name__ + ": " + str(exc)
|
|
event_state_rows.append({"input": freeze(bad) if isinstance(bad, (dict, list)) else bad, "error": error})
|
|
|
|
# ── 11. _find_capability_executor ────────────────────────────────────────────
|
|
|
|
fce_rows = []
|
|
# The list_directory row must NOT list a live system dir (a real /tmp
|
|
# freeze once captured 1.4MB of machine state — the freeze hazard): a
|
|
# versioned repo dir is byte-stable across regens on the same checkout.
|
|
FCE_LIST_DIR = str(ROOT / "schemas" / "mode" / "v1")
|
|
for name, arguments in [
|
|
("read_file", {}),
|
|
("read_file", {"path": "/nonexistent/gap-ltr-probe.txt"}),
|
|
("list_directory", {"path": FCE_LIST_DIR}),
|
|
]:
|
|
result = ltr._find_capability_executor(name, copy.deepcopy(arguments))
|
|
fce_rows.append({"tool": name, "arguments": arguments, "result": freeze(result)})
|
|
|
|
# ── 12. _with_fallback_prompt ────────────────────────────────────────────────
|
|
|
|
SMALL_TOOLS = [
|
|
{"name": "read_file", "description": "Read a file with optional windowed viewing",
|
|
"parameters": {"type": "object", "properties": {"path": {"type": "string", "description": "Absolute path to the file"}, "limit": {"type": "integer"}}, "required": ["path"]}},
|
|
{"name": "list_directory", "description": "List files in a directory",
|
|
"parameters": {"type": "object", "properties": {}, "required": []}},
|
|
]
|
|
|
|
fallback_rows = []
|
|
for messages, label in [
|
|
([{"role": "user", "content": "hello"}], "insert"),
|
|
([{"role": "system", "content": "Be brief."}, {"role": "user", "content": "hello"}], "system-append"),
|
|
]:
|
|
runtime = LocalToolRuntime(transport=FakeTransport([]), executor=ScriptExecutor([]), model="m1", tools=copy.deepcopy(SMALL_TOOLS))
|
|
out = runtime._with_fallback_prompt(copy.deepcopy(messages))
|
|
fallback_rows.append({"label": label, "messages": messages, "result": freeze(out)})
|
|
|
|
# ── 13. resume failure branches ──────────────────────────────────────────────
|
|
|
|
BASE_STATE = {
|
|
"messages": [{"role": "user", "content": "hello"}],
|
|
"native_tools": True,
|
|
"iterations_used": 0,
|
|
"pending_request": {"call_id": "write-1", "name": "write_file", "arguments": {"path": "/tmp/a", "content": "x"}, "source": "ollama-native"},
|
|
}
|
|
|
|
resume_failure_rows = []
|
|
failure_cases = [
|
|
("messages-not-list", {"messages": "nope"}),
|
|
("native-tools-not-bool", {"native_tools": "false"}),
|
|
("options-not-dict", {"options": "x"}),
|
|
("think-not-bool-string", {"think": 5}),
|
|
("exposed-not-list", {"exposed_tools": "nope"}),
|
|
("exposed-not-strings", {"exposed_tools": [1]}),
|
|
("pending-request-missing", {"pending_request": None}),
|
|
("pending-request-not-dict", {"pending_request": "x"}),
|
|
("request-call-id-empty", {"pending_request": {"call_id": "", "name": "write_file", "arguments": {}, "source": "s"}}),
|
|
("request-args-not-dict", {"pending_request": {"call_id": "c", "name": "write_file", "arguments": [], "source": "s"}}),
|
|
("tool-outside-allowlist", {"pending_request": {"call_id": "c", "name": "write_file", "arguments": {"path": "/a", "content": "x"}, "source": "s"}}, {"tools": [{"name": "read_file", "description": "r", "parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}}]}),
|
|
("events-not-list", {"tool_events": "x"}),
|
|
("event-not-dict", {"tool_events": ["x"]}),
|
|
("thinking-not-strings", {"thinking": [1]}),
|
|
("iterations-not-int", {"iterations_used": "3"}),
|
|
("iterations-bool", {"iterations_used": True}),
|
|
("iterations-negative", {"iterations_used": -1}),
|
|
("iterations-over-budget", {"iterations_used": 9}),
|
|
("remaining-not-list", {"remaining_requests": "x"}),
|
|
("remaining-request-bad", {"remaining_requests": ["x"]}),
|
|
]
|
|
for case in failure_cases:
|
|
label, overrides = case[0], case[1]
|
|
tools = case[2]["tools"] if len(case) > 2 else None
|
|
state = copy.deepcopy(BASE_STATE)
|
|
state.update(copy.deepcopy(overrides))
|
|
runtime = LocalToolRuntime(transport=FakeTransport([]), executor=ScriptExecutor([]), model="m1", tools=tools,
|
|
authorize=lambda name: None)
|
|
response = runtime.resume(copy.deepcopy(state), {"success": True, "result": "approved"})
|
|
resume_failure_rows.append({
|
|
"label": label,
|
|
"pending_state": freeze(state),
|
|
"tool_result": {"success": True, "result": "approved"},
|
|
"response": resp_freeze(response),
|
|
})
|
|
|
|
# ── 14. resume loop rows (full envelope + scripts + chat payload parity) ──────
|
|
|
|
resume_loop_rows = []
|
|
|
|
|
|
def make_state(**overrides):
|
|
state = copy.deepcopy(BASE_STATE)
|
|
state.update(copy.deepcopy(overrides))
|
|
return state
|
|
|
|
|
|
def run_resume_row(label, state, tool_result, chat_responses, executor_script, runtime_tools=None):
|
|
"""Construct a fresh runtime with the given scripts, resume, freeze the
|
|
full row (envelope + recorded chat payloads + executor calls + scripts).
|
|
|
|
The runtime carries the PINNED-ALLOW mode authority (the oracle's
|
|
injectable authorize seam): these rows pin the resume/approval
|
|
machinery, not the mode matrix — the mode-gate verdicts have their
|
|
own section below."""
|
|
transport = FakeTransport(list(chat_responses))
|
|
executor = ScriptExecutor(list(executor_script))
|
|
tools = None
|
|
if runtime_tools is not None:
|
|
import rune.tool_executor as te
|
|
tools = [copy.deepcopy(t) for t in te.TOOLS if t["name"] in runtime_tools]
|
|
runtime = LocalToolRuntime(transport=transport, executor=executor, model="m1", tools=tools,
|
|
authorize=lambda name: None)
|
|
response = runtime.resume(copy.deepcopy(state), copy.deepcopy(tool_result))
|
|
return {
|
|
"label": label,
|
|
"pending_state": freeze(state),
|
|
"tool_result": freeze(tool_result),
|
|
"response": wildcard_uuids(resp_freeze(response)),
|
|
"executor_calls": [{"tool": n, "arguments": a} for n, a in executor.calls],
|
|
"executor_script": freeze(executor_script),
|
|
"chat_responses": freeze(chat_responses),
|
|
"runtime_tools": runtime_tools,
|
|
"chat_payloads": wildcard_uuids(freeze(transport.calls)),
|
|
}
|
|
|
|
|
|
TOOL_EVENT_SEED = {
|
|
"kind": "completed",
|
|
"request": {"call_id": "0", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
"result": {"success": True, "result": "map"},
|
|
}
|
|
|
|
# a) read-only pending + remaining → executes, continues, final answer
|
|
state = make_state(
|
|
messages=[{"role": "user", "content": "inspect"}],
|
|
options={"temperature": 0},
|
|
think=True,
|
|
iterations_used=1,
|
|
pending_request={"call_id": "1", "name": "list_directory", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
tool_events=[copy.deepcopy(TOOL_EVENT_SEED)],
|
|
thinking=["thought one"],
|
|
remaining_requests=[{"call_id": "2", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"}],
|
|
exposed_tools=["repo_map", "list_directory", "find_capability"],
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"read-only-continues",
|
|
state,
|
|
{"success": True, "result": "listed"},
|
|
[{"role": "assistant", "content": "All done.", "thinking": "wrap up"}],
|
|
[
|
|
{"success": True, "result": ["a.txt"], "dir": "/tmp"},
|
|
{"success": True, "result": "map body"},
|
|
],
|
|
))
|
|
|
|
# b) model issues a NEW mutating call → plain needs_confirm approval state
|
|
# (executor without tool authority); the resume tool_result only settles
|
|
# the pending request.
|
|
state = make_state(
|
|
messages=[{"role": "user", "content": "write it"}],
|
|
think=None,
|
|
iterations_used=0,
|
|
pending_request={"call_id": "w1", "name": "write_file", "arguments": {"path": "/tmp/gap-ltr.txt", "content": "x"}, "source": "ollama-native"},
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"pending-approval",
|
|
state,
|
|
{"success": True, "result": "approved"},
|
|
[
|
|
{"role": "assistant", "content": "", "tool_calls": [{"id": "w2", "function": {"name": "write_file", "arguments": {"path": "/tmp/gap-ltr.txt", "content": "y"}}}]},
|
|
],
|
|
[],
|
|
))
|
|
|
|
# c) rejected tool result → error event, loop continues to final
|
|
state = make_state(
|
|
messages=[{"role": "user", "content": "write it"}],
|
|
iterations_used=0,
|
|
pending_request={"call_id": "w1", "name": "write_file", "arguments": {"path": "/tmp/gap-ltr.txt", "content": "x"}, "source": "ollama-native"},
|
|
thinking=["decided"],
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"rejected-result",
|
|
state,
|
|
{"success": False, "error": "operator rejected"},
|
|
[{"role": "assistant", "content": "Left unchanged."}],
|
|
[],
|
|
))
|
|
|
|
# d) circuit breaker: two consecutive identical IN-LOOP dispatch failures
|
|
# (the resume tool_result settles the pending request and is NOT
|
|
# registered with the breaker — signature tracking starts in the loop).
|
|
state = make_state(
|
|
pending_request={"call_id": "1", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"circuit-breaker",
|
|
state,
|
|
{"success": False, "error": "no such dir"},
|
|
[
|
|
{"role": "assistant", "content": "", "tool_calls": [{"id": "2", "function": {"name": "repo_map", "arguments": {"path": "/x"}}}]},
|
|
{"role": "assistant", "content": "", "tool_calls": [{"id": "3", "function": {"name": "repo_map", "arguments": {"path": "/x"}}}]},
|
|
],
|
|
[
|
|
{"success": False, "error": "no such dir"},
|
|
{"success": False, "error": "no such dir"},
|
|
],
|
|
))
|
|
|
|
# e) iteration limit after resume consumed budget
|
|
state = make_state(
|
|
pending_request={"call_id": "1", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
iterations_used=8,
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"iteration-limit",
|
|
state,
|
|
{"success": True, "result": "ok"},
|
|
[],
|
|
[{"success": True, "result": "map"}],
|
|
))
|
|
|
|
# f) text-fallback resume (native_tools false)
|
|
state = make_state(
|
|
messages=[{"role": "user", "content": "inspect"}],
|
|
native_tools=False,
|
|
pending_request={"call_id": "1", "name": "list_directory", "arguments": {"path": "/tmp"}, "source": "text-fallback"},
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"text-fallback",
|
|
state,
|
|
{"success": True, "result": "listed"},
|
|
[
|
|
{"role": "assistant", "content": 'blah {"tool":"list_directory","args":{"path":"/tmp"}} blah'},
|
|
{"role": "assistant", "content": "fallback done"},
|
|
],
|
|
[
|
|
{"success": True, "result": ["a.txt"]},
|
|
{"success": True, "result": ["a.txt"]},
|
|
],
|
|
))
|
|
|
|
# g) tool outside allowlist during loop → escape-hatch message
|
|
state = make_state(
|
|
pending_request={"call_id": "1", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"allowlist-escape-hatch",
|
|
state,
|
|
{"success": True, "result": "ok"},
|
|
[
|
|
{"role": "assistant", "content": "", "tool_calls": [{"id": "2", "function": {"name": "write_file", "arguments": {"path": "/a", "content": "x"}}}]},
|
|
],
|
|
[{"success": True, "result": "map"}],
|
|
runtime_tools=("repo_map", "list_directory", "read_file"),
|
|
))
|
|
|
|
# h) malformed native tool calls → structured failure
|
|
state = make_state(
|
|
pending_request={"call_id": "1", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"malformed-args-json",
|
|
state,
|
|
{"success": True, "result": "ok"},
|
|
[
|
|
{"role": "assistant", "content": "", "tool_calls": [{"id": "2", "function": {"name": "repo_map", "arguments": "{bad"}}]},
|
|
],
|
|
[{"success": True, "result": "map"}],
|
|
))
|
|
|
|
# i) empty terminal visible text
|
|
state = make_state(
|
|
pending_request={"call_id": "1", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"empty-terminal",
|
|
state,
|
|
{"success": True, "result": "ok"},
|
|
[{"role": "assistant", "content": " \n\t"}],
|
|
[{"success": True, "result": "map"}],
|
|
))
|
|
|
|
# j) internal-protocol marker not at line start is ordinary content
|
|
state = make_state(
|
|
pending_request={"call_id": "1", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"internal-protocol",
|
|
state,
|
|
{"success": True, "result": "ok"},
|
|
[{"role": "assistant", "content": "here [tool result] leaked"}],
|
|
[{"success": True, "result": "map"}],
|
|
))
|
|
|
|
# k) unknown tool in native calls → escape hatch (validate inside normalize)
|
|
state = make_state(
|
|
pending_request={"call_id": "1", "name": "repo_map", "arguments": {"path": "/tmp"}, "source": "ollama-native"},
|
|
)
|
|
resume_loop_rows.append(run_resume_row(
|
|
"unknown-tool-native",
|
|
state,
|
|
{"success": True, "result": "ok"},
|
|
[
|
|
{"role": "assistant", "content": "", "tool_calls": [{"id": "2", "function": {"name": "definitely_not_a_tool", "arguments": {}}}]},
|
|
],
|
|
[{"success": True, "result": "map"}],
|
|
))
|
|
|
|
# ── 15. mode-gate rows (the T3 two-door law) ──────────────────────────────────
|
|
# NO pinned-allow authority here: the live default (mode_gate.authorize_tool)
|
|
# resolves the active mode per call. RUNE_MODE is pinned per row so the
|
|
# freeze is deterministic (env wins over any hot-state file).
|
|
|
|
mode_gate_rows = []
|
|
|
|
WRITE_CALL = {"role": "assistant", "content": "", "tool_calls": [
|
|
{"id": "w1", "function": {"name": "write_file", "arguments": {"path": "/tmp/gap-ltr.txt", "content": "y"}}},
|
|
]}
|
|
|
|
|
|
def run_mode_row(label, mode_env, chat_responses, executor_script):
|
|
old = os.environ.get("RUNE_MODE")
|
|
os.environ["RUNE_MODE"] = mode_env
|
|
try:
|
|
transport = FakeTransport(list(chat_responses))
|
|
executor = ScriptExecutor(list(executor_script))
|
|
runtime = LocalToolRuntime(transport=transport, executor=executor, model="m1")
|
|
response = runtime.run([{"role": "user", "content": "go"}])
|
|
finally:
|
|
if old is None:
|
|
os.environ.pop("RUNE_MODE", None)
|
|
else:
|
|
os.environ["RUNE_MODE"] = old
|
|
mode_gate_rows.append({
|
|
"label": label,
|
|
"mode_env": mode_env,
|
|
"response": wildcard_uuids(resp_freeze(response)),
|
|
"executor_calls": [{"tool": n, "arguments": a} for n, a in executor.calls],
|
|
"chat_responses": freeze(chat_responses),
|
|
"chat_payloads": wildcard_uuids(freeze(transport.calls)),
|
|
})
|
|
|
|
|
|
# a) CHAT-originated write is mode-refused as typed tool-error data; the
|
|
# loop answers the model and the executor NEVER runs.
|
|
run_mode_row(
|
|
"chat-write-refused",
|
|
"CHAT",
|
|
[copy.deepcopy(WRITE_CALL), {"role": "assistant", "content": "understood"}],
|
|
[],
|
|
)
|
|
|
|
# b) repeated identical refusals trip the failure circuit breaker.
|
|
run_mode_row(
|
|
"chat-write-repeat-breaker",
|
|
"CHAT",
|
|
[copy.deepcopy(WRITE_CALL),
|
|
{"role": "assistant", "content": "", "tool_calls": [
|
|
{"id": "w2", "function": {"name": "write_file", "arguments": {"path": "/tmp/gap-ltr.txt", "content": "y"}}}]}],
|
|
[],
|
|
)
|
|
|
|
# c) TASK allows the write family: the call reaches the approval door
|
|
# (plain needs_confirm — this executor has no tool authority).
|
|
run_mode_row(
|
|
"task-write-reaches-approval",
|
|
"TASK",
|
|
[copy.deepcopy(WRITE_CALL)],
|
|
[],
|
|
)
|
|
|
|
# d) an invalid RUNE_MODE fails CLOSED to CHAT (env-invalid source).
|
|
run_mode_row(
|
|
"invalid-mode-fails-closed",
|
|
"BOGUS",
|
|
[copy.deepcopy(WRITE_CALL), {"role": "assistant", "content": "understood"}],
|
|
[],
|
|
)
|
|
|
|
# ── write ─────────────────────────────────────────────────────────────────────
|
|
|
|
truth = {
|
|
"extract_context_length": extract_rows,
|
|
"positive_int": positive_int_rows,
|
|
"model_capabilities": cap_rows,
|
|
"transport_context_length": tctx_rows,
|
|
"module_context_length": module_ctx_rows,
|
|
"reset_cache": reset_row,
|
|
"deduplicate": dedup_rows,
|
|
"tool_message": tool_message_rows,
|
|
"request_state": request_state_rows,
|
|
"event_state": event_state_rows,
|
|
"find_capability_executor": fce_rows,
|
|
"fallback_prompt": fallback_rows,
|
|
"resume_failures": resume_failure_rows,
|
|
"resume_loops": resume_loop_rows,
|
|
"mode_gate": mode_gate_rows,
|
|
}
|
|
|
|
out = ROOT / "src" / "go" / "local-tool-runtime" / "truth.json"
|
|
out.write_text(json.dumps(truth, indent=1, sort_keys=True) + "\n")
|
|
print("written:", out, len(json.dumps(truth)), "bytes")
|
|
for key, rows in truth.items():
|
|
print(f" {key}: {len(rows) if isinstance(rows, list) else 1}")
|