Nick
8a08f9b4be
CI Pipeline / best-practices (push) Successful in 11s
CI Pipeline / documentation (push) Successful in 3s
CI Pipeline / rust-surface (push) Failing after 17s
CI Pipeline / go-surface (push) Failing after 20s
CI Pipeline / gitea-offline (push) Successful in 16s
CI Pipeline / required-checks (push) Canceled after 0s
CI Pipeline / CI Pipeline / notify-event-bridge (push) Canceled after 0s
CI Pipeline / type-check (push) Canceled after 29s
deploy / deploy (push) Canceled after 47s
Auto-version and Release / release (push) Canceled after 50s
CI Pipeline / lint (push) Canceled after 45s
CI Pipeline / unit-tests (push) Canceled after 45s
CI Pipeline / canary (push) Canceled after 46s
107 lines
3.2 KiB
Python
107 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Live parity — provider_budget python oracle vs the GO port (gap-closing
|
|
wave #2479). Random op script (seed 2479: track_usage / failover / best /
|
|
status over the fixture pool) applied to BOTH implementations against the
|
|
same canonical fixture home; get_status after every op must match.
|
|
|
|
check_provider is excluded (live sockets + wall-clock stamps are reactive
|
|
surface, pinned by the Go seam battery instead).
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
sys.path.insert(0, str(REPO / "scripts"))
|
|
|
|
import gen_provider_budget_truth as gen # noqa: E402
|
|
|
|
SEED = 2479
|
|
OPS = 200
|
|
GO_DIR = REPO / "src" / "go" / "provider-budget"
|
|
|
|
|
|
def build_ops() -> list[dict]:
|
|
rng = random.Random(SEED)
|
|
names = ["alpha", "beta", "gamma", "delta", "ghost"]
|
|
ops = []
|
|
for _ in range(OPS):
|
|
kind = rng.choices(
|
|
["track_usage", "failover", "best", "status"], weights=[6, 2, 1, 1]
|
|
)[0]
|
|
if kind == "track_usage":
|
|
ops.append({
|
|
"op": "track_usage",
|
|
"name": rng.choice(names),
|
|
"tokens": rng.randrange(0, 2000),
|
|
"cost": round(rng.uniform(0, 3), 4),
|
|
})
|
|
elif kind == "failover":
|
|
ops.append({"op": "failover", "name": rng.choice(names)})
|
|
else:
|
|
ops.append({"op": kind})
|
|
return ops
|
|
|
|
|
|
def run_python(ops: list[dict]) -> list[dict]:
|
|
gen.build_fixture_home()
|
|
os.environ["HOME"] = str(gen.HOME)
|
|
os.environ["RUNE_PROVIDER_BUDGET_CORE"] = "python" # poisoned-truth law
|
|
import rune.provider_budget as pb
|
|
|
|
pb._tracker = None
|
|
want = []
|
|
for op in ops:
|
|
kind = op["op"]
|
|
if kind == "track_usage":
|
|
pb.track_usage(op["name"], op["tokens"], op["cost"])
|
|
elif kind == "failover":
|
|
pb.failover(op["name"])
|
|
elif kind == "best":
|
|
pb.get_best_provider()
|
|
elif kind == "status":
|
|
pb.get_status()
|
|
else:
|
|
raise ValueError(f"unknown op {kind}")
|
|
want.append(pb.get_status())
|
|
return want
|
|
|
|
|
|
def run_go(ops: list[dict]) -> list[dict]:
|
|
gen.build_fixture_home()
|
|
ops_file = Path("/tmp/rune_pb_parity_ops.json")
|
|
ops_file.write_text(json.dumps(ops))
|
|
r = subprocess.run(
|
|
["go", "run", ".", "parity", str(ops_file), str(gen.HOME)],
|
|
cwd=GO_DIR, capture_output=True, text=True,
|
|
)
|
|
if r.returncode != 0:
|
|
print(r.stderr, file=sys.stderr)
|
|
sys.exit(1)
|
|
return [json.loads(line) for line in r.stdout.splitlines() if line.strip()]
|
|
|
|
|
|
def main() -> int:
|
|
ops = build_ops()
|
|
want = run_python(ops)
|
|
got = run_go(ops)
|
|
fails = 0
|
|
for i, (w, g) in enumerate(zip(want, got)):
|
|
if w != g:
|
|
fails += 1
|
|
if fails <= 3:
|
|
print(f"op {i} ({ops[i]['op']}) diverged:")
|
|
print(f" want {json.dumps(w, sort_keys=True)[:240]}")
|
|
print(f" got {json.dumps(g, sort_keys=True)[:240]}")
|
|
print(f"parity: {len(want) - fails}/{len(want)} ops identical (seed {SEED})")
|
|
return 1 if fails else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|