Nick
4032c753f3
The model→tool boundary ported to tool-protocol: - ValidateToolRequest: alias canonicalization BEFORE schema (grep→ grep_files, bash→run_command — bash never bypasses run_command's approval boundary), exact Python error texts (unknown tool / required argument missing / unknown argument / must be <type>), array item type checks, deterministic arg iteration for error stability - schemas from the FROZEN tools_registry.json (shared with tool-executor — the set-equality authority) - NormalizeOllamaToolCalls: native envelope + string-arguments JSON decode, malformed-shape errors - ContainsInternalProtocol: result/error markers + xml tool_call spans - ToolRequest dataclass aligned to the Python shape (Arguments field) - 17-case truth battery from the live oracle; span-surgery surfaces (extract_visible_text, text-fallback parse) pinned for batch 2
95 lines
4.5 KiB
Python
95 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""gen_tool_protocol_truth.py — factory batch 1 truth battery
|
|
(contract BUILDER-2026-10-07T22:11:52Z-9ffb73).
|
|
|
|
Live tool_protocol oracle fixtures: validate error texts, alias
|
|
canonicalization, ollama envelope normalization (incl. string-arguments
|
|
JSON), internal-protocol detection, extract_visible_text span stripping.
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
sys.path.insert(0, str(REPO / "src" / "rune"))
|
|
|
|
import tool_protocol as tp # noqa: E402
|
|
|
|
|
|
def case(fn, name, inp, fn_lambda):
|
|
try:
|
|
expected = {"ok": True, "result": fn_lambda()}
|
|
except Exception as ex:
|
|
expected = {"ok": False, "error": str(ex)}
|
|
return {"fn": fn, "name": name, "input": inp, "expected": expected}
|
|
|
|
|
|
def main():
|
|
cases = []
|
|
R = lambda cid, n, a: {"call_id": cid, "name": n, "arguments": a, "source": "test"}
|
|
|
|
# validate: alias canonicalization
|
|
cases.append(case("validate", "alias_grep",
|
|
{"request": R("c1", "grep", {"pattern": "x", "path": "."})},
|
|
lambda: tp.validate_tool_request(tp.ToolRequest(**R("c1", "grep", {"pattern": "x", "path": "."}))).to_dict()))
|
|
cases.append(case("validate", "alias_bash",
|
|
{"request": R("c2", "bash", {"command": "ls"})},
|
|
lambda: tp.validate_tool_request(tp.ToolRequest(**R("c2", "bash", {"command": "ls"}))).to_dict()))
|
|
cases.append(case("validate", "unknown_tool",
|
|
{"request": R("c3", "nonexistent_tool", {})},
|
|
lambda: tp.validate_tool_request(tp.ToolRequest(**R("c3", "nonexistent_tool", {}))).to_dict()))
|
|
cases.append(case("validate", "missing_required",
|
|
{"request": R("c4", "read_file", {})},
|
|
lambda: tp.validate_tool_request(tp.ToolRequest(**R("c4", "read_file", {}))).to_dict()))
|
|
cases.append(case("validate", "unknown_arg",
|
|
{"request": R("c5", "read_file", {"path": "/tmp", "bogus": 1})},
|
|
lambda: tp.validate_tool_request(tp.ToolRequest(**R("c5", "read_file", {"path": "/tmp", "bogus": 1}))).to_dict()))
|
|
cases.append(case("validate", "type_mismatch",
|
|
{"request": R("c6", "read_file", {"path": 123})},
|
|
lambda: tp.validate_tool_request(tp.ToolRequest(**R("c6", "read_file", {"path": 123}))).to_dict()))
|
|
|
|
# ollama normalization: string arguments JSON
|
|
cases.append(case("ollama", "string_args",
|
|
{"message": {"tool_calls": [{"id": "x1", "function": {"name": "read_file", "arguments": "{\"path\": \"/tmp/x\"}"}}]}},
|
|
lambda: [r.to_dict() for r in tp.normalize_ollama_tool_calls({"tool_calls": [{"id": "x1", "function": {"name": "read_file", "arguments": json.dumps({"path": "/tmp/x"})}}]})]))
|
|
cases.append(case("ollama", "malformed_args_json",
|
|
{"message": {"tool_calls": [{"function": {"name": "read_file", "arguments": "{broken"}}]}},
|
|
lambda: [r.to_dict() for r in tp.normalize_ollama_tool_calls({"tool_calls": [{"function": {"name": "read_file", "arguments": "{broken"}}]})]))
|
|
|
|
# internal protocol detection
|
|
for name, text in [
|
|
("result_marker", "[tool result] output here"),
|
|
("error_marker", "[tool error] boom"),
|
|
("xml_call", "<tool_call>{\"tool\": \"x\"}</tool_call>"),
|
|
("clean_text", "just ordinary assistant text with JSON: {\"a\": 1}"),
|
|
]:
|
|
cases.append(case("internal", name, {"text": text},
|
|
lambda t=text: {"contains": tp.contains_internal_protocol(t)}))
|
|
|
|
# extract_visible_text
|
|
for name, text in [
|
|
("inline_call", 'before {\"tool\": \"read_file\", \"args\": {\"path\": \"/tmp\"}} after'),
|
|
("fenced_call", '```json\n{\"tool\": \"read_file\", \"args\": {\"path\": \"/tmp\"}}\n```'),
|
|
("xml_call", 'before <tool_call>{\"tool\": \"x\", \"args\": {}}</tool_call> after'),
|
|
("plain", "no protocol here at all"),
|
|
]:
|
|
cases.append(case("visible", name, {"text": text},
|
|
lambda t=text: {"visible": tp.extract_visible_text(t)}))
|
|
|
|
# normalize_text_tool_calls end-to-end
|
|
cases.append(case("normalize", "text_fallback",
|
|
{"text": '{\"tool\": \"read_file\", \"args\": {\"path\": \"/tmp/x\"}}'},
|
|
lambda: [r.to_dict() for r in tp.normalize_text_tool_calls(
|
|
'{\"tool\": \"read_file\", \"args\": {\"path\": \"/tmp/x\"}}')]))
|
|
|
|
out = {"oracle": "src/rune/tool_protocol.py (live)", "generated": "2026-10-07", "cases": cases}
|
|
target = REPO / "src/go/tool-protocol" / "truth_protocol.json"
|
|
target.write_text(json.dumps(out, indent=1, ensure_ascii=False) + "\n")
|
|
print(f"wrote {target} ({len(cases)} cases)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|