Nick
df6f52e652
- tool-executor batch 6: jsonUnmarshal/paramOrder/rawKeyOrder moved from
truth_prompt_test.go into json_order.go — the non-test build failed with
undefined symbols (go-surface run 19792 job 56712)
- truth batteries regenerated HERMETICALLY via the committed live-oracle
generator (scripts/gen_tool_executor_truth.py): {{TMP}}/{{WROOT}}
placeholders replace the generation host's /mnt/workdrive/tmp* paths in
inputs AND expectations; fixtures manifest carried in each truth file;
tests recreate fixtures under their own allowed roots (truth_harness_test.go)
- list_dir comparison now exact (both sides sort); grep compares sorted
line sets (oracle os.walk order is OS-dependent, not a parity property);
truncation marker pinned by a new hermetic test
- construction_decision lib.rs: the 'rune dichotomy:' rule-string parity
half committed (Python half landed in 6d63034; strings verified identical)
- stale in-tree binaries rebuilt from current source
302 lines
11 KiB
Python
302 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""gen_tool_executor_truth.py — regenerate the tool-executor truth batteries
|
|
HERMETICALLY (contract BUILDER-2026-10-07T16:11:35Z-70ca24-unknown).
|
|
|
|
Why: the original batch-2/3/4 generators baked the generation host's temp
|
|
paths (/mnt/workdrive/tmp*) into truth_fileops.json / truth_ops.json /
|
|
truth_paths.json — inputs AND expectations. CI's runner user cannot read
|
|
those paths, so the parity gate failed with permission-denied (go-surface,
|
|
run 19792 job 56712) despite a locally green tree.
|
|
|
|
Design (the environment leak, closed):
|
|
- Fixtures are created at generation time under roots the ORACLE ITSELF
|
|
allows: read root = first creatable dir in ALLOWED_READ_DIRS (prefers
|
|
/tmp), write root = first creatable dir in ALLOWED_WRITE_DIRS.
|
|
- Every occurrence of those roots — in inputs AND expectations — is
|
|
replaced with {{TMP}} / {{WROOT}} placeholders.
|
|
- Each truth file carries a `fixtures` manifest (files + dirs) so the Go
|
|
test recreates the SAME fixture content under ITS OWN roots at runtime.
|
|
Both sides then run the same case semantics on their own environment;
|
|
substituted bytes must agree exactly.
|
|
- The grep-hit comparison stays order-insensitive (sorted line sets) on the
|
|
Go side: the oracle's os.walk order is OS-scheduling dependent, not a
|
|
parity-relevant property. Everything else is exact after substitution.
|
|
|
|
Oracle authority: src/rune/tool_executor.py is called LIVE for every case
|
|
except the two cmd_gate verdicts whose oracle path would execute a shell
|
|
command (plain_ok / shell_ok use harmless commands instead: "ls -la" and
|
|
"true" — the gate branches they exercise do not inspect command content
|
|
beyond non-emptiness).
|
|
|
|
Usage: python3 scripts/gen_tool_executor_truth.py (from repo root)
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
|
|
from rune import tool_executor as te # noqa: E402 — the live oracle
|
|
|
|
|
|
def mk_root(dirs, tag):
|
|
"""First creatable temp dir under the given allowed dirs."""
|
|
for d in dirs:
|
|
try:
|
|
return tempfile.mkdtemp(prefix="rune-truth-", dir=d)
|
|
except OSError:
|
|
continue
|
|
raise RuntimeError(f"no creatable root among {dirs}")
|
|
|
|
|
|
def build_fixtures(root):
|
|
"""The shared fixture set: exactly what the cases observe."""
|
|
(Path(root) / "w.txt").write_text(
|
|
"".join(f"line {i}\n" for i in range(1, 21)), encoding="utf-8"
|
|
)
|
|
(Path(root) / "a.txt").write_text("the spine rebuild parity\n", encoding="utf-8")
|
|
(Path(root) / "b.log").write_text("spine appears here too\n", encoding="utf-8")
|
|
(Path(root) / "file.txt").write_text("content\n", encoding="utf-8")
|
|
(Path(root) / "adir").mkdir()
|
|
return {
|
|
"files": [
|
|
{"path": "{{TMP}}/w.txt",
|
|
"content": "".join(f"line {i}\n" for i in range(1, 21))},
|
|
{"path": "{{TMP}}/a.txt", "content": "the spine rebuild parity\n"},
|
|
{"path": "{{TMP}}/b.log", "content": "spine appears here too\n"},
|
|
{"path": "{{TMP}}/file.txt", "content": "content\n"},
|
|
],
|
|
"dirs": ["{{TMP}}/adir"],
|
|
}
|
|
|
|
|
|
def main():
|
|
read_root = mk_root(te.ALLOWED_READ_DIRS, "read")
|
|
write_root = mk_root(te.ALLOWED_WRITE_DIRS, "write")
|
|
fixtures = build_fixtures(read_root)
|
|
|
|
wpath = f"{read_root}/w.txt"
|
|
edited = "line 1\nCHANGED\n" + "".join(f"line {i}\n" for i in range(3, 21))
|
|
|
|
cases_fileops = [
|
|
{
|
|
"fn": "read_window", "name": "mid",
|
|
"input": {"path": f"{wpath}", "offset": 3, "limit": 5},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "read_window", "name": "default",
|
|
"input": {"path": f"{wpath}"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "diff", "name": "simple",
|
|
"input": {"path": wpath, "new_content": edited},
|
|
"expected": None,
|
|
},
|
|
]
|
|
|
|
r = te._read_file({"path": wpath, "offset": 3, "limit": 5})
|
|
assert r["success"] is True, r
|
|
cases_fileops[0]["expected"] = {"ok": True, "result": r["result"]}
|
|
|
|
r = te._read_file({"path": wpath})
|
|
assert r["success"] is True, r
|
|
cases_fileops[1]["expected"] = {
|
|
"ok": True, "lines": len(r["result"].split("\n")),
|
|
}
|
|
|
|
diff = te.generate_diff(Path(wpath), edited)
|
|
dlines = diff.split("\n")
|
|
cases_fileops[2]["expected"] = {
|
|
"diff_head": dlines[:3],
|
|
"has_minus": any(l.startswith("-line 2") for l in dlines),
|
|
"has_plus": any(l.startswith("+CHANGED") for l in dlines),
|
|
}
|
|
|
|
cases_ops = [
|
|
{
|
|
"fn": "list_dir", "name": "basic",
|
|
"input": {"path": read_root},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "list_dir", "name": "pattern",
|
|
"input": {"path": read_root, "pattern": "*.txt"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "grep", "name": "hit",
|
|
"input": {"path": read_root, "pattern": "spine"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "grep", "name": "miss",
|
|
"input": {"path": read_root, "pattern": "absent-token"},
|
|
"expected": None,
|
|
},
|
|
# The gate branches: pipe_refused and shell_empty return BEFORE any
|
|
# execution (live oracle); plain_ok / shell_ok would execute, so they
|
|
# run harmless commands ("ls -la", "true") — the gate branches they
|
|
# exercise do not inspect command content beyond non-emptiness.
|
|
{
|
|
"fn": "cmd_gate", "name": "pipe_refused",
|
|
"input": {"command": "ls -la | head"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "cmd_gate", "name": "plain_ok",
|
|
"input": {"command": "ls -la"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "cmd_gate", "name": "shell_ok",
|
|
"input": {"command": "true", "shell": True},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "cmd_gate", "name": "shell_empty",
|
|
"input": {"shell": True},
|
|
"expected": None,
|
|
},
|
|
]
|
|
|
|
r = te._list_directory({"path": read_root})
|
|
assert r["success"] is True, r
|
|
cases_ops[0]["expected"] = {"ok": True, "result": r["result"]}
|
|
|
|
r = te._list_directory({"path": read_root, "pattern": "*.txt"})
|
|
assert r["success"] is True, r
|
|
cases_ops[1]["expected"] = {"ok": True, "result": r["result"]}
|
|
|
|
r = te._search_files({"path": read_root, "pattern": "spine"})
|
|
assert r["success"] is True and r["result"] != "(no matches)", r
|
|
cases_ops[2]["expected"] = {"ok": True, "result": r["result"]}
|
|
|
|
r = te._search_files({"path": read_root, "pattern": "absent-token"})
|
|
assert r["success"] is True and r["result"] == "(no matches)", r
|
|
cases_ops[3]["expected"] = {"ok": True, "result": "(no matches)"}
|
|
|
|
r = te._run_command({"command": "ls -la | head"})
|
|
cases_ops[4]["expected"] = {"success": r["success"], "error": r.get("error", "")}
|
|
assert r["success"] is False, r
|
|
|
|
r = te._run_command({"command": "ls -la"})
|
|
cases_ops[5]["expected"] = {"success": r["success"], "error": r.get("error", "")}
|
|
assert r["success"] is True, r
|
|
|
|
r = te._run_command({"command": "true", "shell": True})
|
|
cases_ops[6]["expected"] = {"success": r["success"], "error": r.get("error", "")}
|
|
assert r["success"] is True, r
|
|
|
|
r = te._run_command({"shell": True})
|
|
cases_ops[7]["expected"] = {"success": r["success"], "error": r.get("error", "")}
|
|
assert r["success"] is False, r
|
|
|
|
cases_paths = [
|
|
{
|
|
"fn": "resolve", "name": "abs",
|
|
"input": {"path": f"{read_root}/file.txt"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "validate_read", "name": "hit",
|
|
"input": {"path": f"{read_root}/file.txt"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "validate_read", "name": "miss",
|
|
"input": {"path": f"{read_root}/missing.txt"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "validate_read", "name": "blocked",
|
|
"input": {"path": "/etc/hostname"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "validate_write", "name": "protected",
|
|
"input": {"path": "/etc/x.txt"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "validate_write", "name": "outside",
|
|
"input": {"path": "/usr/share/x.txt"},
|
|
"expected": None,
|
|
},
|
|
{
|
|
"fn": "validate_write", "name": "ok",
|
|
"input": {"path": f"{write_root}/new.txt"},
|
|
"expected": None,
|
|
},
|
|
]
|
|
|
|
resolved = str(te.resolve_path(f"{read_root}/file.txt"))
|
|
cases_paths[0]["expected"] = {"resolved": resolved}
|
|
|
|
ok, msg, _ = te.validate_read_path(f"{read_root}/file.txt")
|
|
cases_paths[1]["expected"] = {"ok": ok}
|
|
assert ok is True, msg
|
|
|
|
ok, msg, _ = te.validate_read_path(f"{read_root}/missing.txt")
|
|
cases_paths[2]["expected"] = {"ok": ok, "msg": msg}
|
|
assert ok is False, msg
|
|
|
|
ok, msg, _ = te.validate_read_path("/etc/hostname")
|
|
cases_paths[3]["expected"] = {"ok": ok}
|
|
assert ok is False, msg
|
|
|
|
ok, msg, _ = te.validate_write_path("/etc/x.txt")
|
|
cases_paths[4]["expected"] = {"ok": ok, "msg_prefix": msg[:50]}
|
|
assert ok is False, msg
|
|
|
|
ok, msg, _ = te.validate_write_path("/usr/share/x.txt")
|
|
cases_paths[5]["expected"] = {"ok": ok, "msg_prefix": msg[:50]}
|
|
assert ok is False, msg
|
|
|
|
ok, msg, resolved = te.validate_write_path(f"{write_root}/new.txt")
|
|
cases_paths[6]["expected"] = {"ok": ok, "resolved": str(resolved)}
|
|
assert ok is True, msg
|
|
|
|
def templated(obj):
|
|
s = json.dumps(obj, ensure_ascii=False)
|
|
s = s.replace(read_root, "{{TMP}}").replace(write_root, "{{WROOT}}")
|
|
return json.loads(s)
|
|
|
|
out = {
|
|
"oracle": "src/rune/tool_executor.py",
|
|
"generated": "2026-10-07 (hermetic regeneration)",
|
|
"placeholders": {
|
|
"{{TMP}}": "read-fixture root (first creatable ALLOWED_READ_DIRS entry)",
|
|
"{{WROOT}}": "write root (first creatable ALLOWED_WRITE_DIRS entry)",
|
|
},
|
|
"fixtures": fixtures,
|
|
"cases": None,
|
|
}
|
|
|
|
for name, cases in (
|
|
("truth_fileops.json", cases_fileops),
|
|
("truth_ops.json", cases_ops),
|
|
("truth_paths.json", cases_paths),
|
|
):
|
|
doc = dict(out)
|
|
doc["cases"] = templated(cases)
|
|
text = json.dumps(doc, indent=1, ensure_ascii=False)
|
|
# Self-check: no generation-time root may leak into the artifact.
|
|
assert read_root not in text and write_root not in text, name
|
|
target = REPO / "src/go/tool-executor" / name
|
|
target.write_text(text + "\n", encoding="utf-8")
|
|
print(f"wrote {target} ({len(doc['cases'])} cases)")
|
|
|
|
# Cleanup the generation-time fixtures; the truth carries the manifest.
|
|
import shutil
|
|
shutil.rmtree(read_root, ignore_errors=True)
|
|
shutil.rmtree(write_root, ignore_errors=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|