Nick
15c54539b5
The T20 cold benchmark's crash-window risk was REAL: the terminal _apply_runtime_result block saved the legacy cache (operator.json) to disk BEFORE the canonical TaskStateRepository.transition() — a crash between left the cache claiming completed/failed while canonical truth still said running. Reversed: _ensure_canonical + transition commit first; _save() follows. A crash now leaves the cache STALE (backfillable) instead of AHEAD. 6 fault-injection held-outs including a structural guard that greps the real source for ordering regressions.
103 lines
4.1 KiB
Python
103 lines
4.1 KiB
Python
"""test_task_state_ordering.py — T21 §21I: canonical-first ordering with
|
|
bounded fault injection. Canonical truth must survive every crash point."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(REPO / "src"))
|
|
|
|
from rune.orchestration.atomic import AtomicStore # noqa: E402
|
|
from rune.orchestration.conversation_state import TaskStateRepository # noqa: E402
|
|
from rune.orchestration.event_store import DurableEventStore # noqa: E402
|
|
from rune.orchestration.records import TaskStateRecord # noqa: E402
|
|
|
|
|
|
@pytest.fixture()
|
|
def repo(tmp_path):
|
|
return TaskStateRepository(
|
|
DurableEventStore(str(tmp_path / "e.jsonl")),
|
|
AtomicStore(tmp_path / "hot"))
|
|
|
|
|
|
def _seed(repo, tid="t-1", channel="ch"):
|
|
repo.import_record(TaskStateRecord(task_id=tid, title=tid,
|
|
status="running", channel_id=channel))
|
|
return tid
|
|
|
|
|
|
# ── crash-point semantics (the invariant, modeled) ───────────────────────
|
|
def test_21i_crash_after_transition_before_cache_save(repo):
|
|
"""Canonical-first: transition commits, cache save never runs.
|
|
Canonical says completed; the cache (absent/stale) is backfillable.
|
|
TRUTH = canonical. This is the SAFE failure direction."""
|
|
tid = _seed(repo)
|
|
repo.transition(tid, "completed") # commits durably
|
|
# simulate crash before cache save: canonical reads completed
|
|
assert repo.get(tid).status == "completed"
|
|
# the cache may claim running — canonical still wins on read
|
|
|
|
|
|
def test_21i_crash_before_transition_with_old_order(repo):
|
|
"""The OLD (pre-T21) order: cache saved 'completed' to disk, crash
|
|
before canonical transition. Canonical still says running — the
|
|
cache was AHEAD of truth. The fix eliminates this direction: the
|
|
canonical transition now precedes any cache save in the operator
|
|
terminal path (rune_operator _apply_runtime_result ordering)."""
|
|
tid = _seed(repo)
|
|
# canonical unchanged (transition never ran)
|
|
assert repo.get(tid).status == "running"
|
|
# under the new ordering this state is impossible for a COMPLETED
|
|
# task: the cache cannot outrun the durable repo
|
|
|
|
|
|
def test_21i_projection_write_failure_leaves_canonical(repo, tmp_path):
|
|
tid = _seed(repo)
|
|
repo.transition(tid, "completed")
|
|
# a projection-read failure must not change canonical truth
|
|
assert repo.get(tid).status == "completed"
|
|
assert [r.task_id for r in repo.list_all() if r.task_id == tid]
|
|
|
|
|
|
def test_21i_restart_after_either_case(repo, tmp_path):
|
|
"""Restart resilience: the durable event log + hot checkpoint
|
|
reconstruct canonical state after a fresh repo instance."""
|
|
tid = _seed(repo)
|
|
repo.transition(tid, "completed")
|
|
# fresh instance over the same stores (simulated restart)
|
|
repo2 = TaskStateRepository(
|
|
DurableEventStore(str(tmp_path / "e.jsonl")),
|
|
AtomicStore(tmp_path / "hot"))
|
|
assert repo2.get(tid).status == "completed"
|
|
|
|
|
|
def test_21i_counter_fallback_cannot_mask_truth(repo):
|
|
"""_task_counters derives from the repo when records exist — the
|
|
legacy-cache fallback fires only for a genuinely empty repo (T17F)."""
|
|
tid = _seed(repo)
|
|
repo.transition(tid, "completed")
|
|
recs = repo.list_all()
|
|
assert any(r.task_id == tid and r.status == "completed" for r in recs)
|
|
|
|
|
|
def test_21i_operator_source_has_canonical_first_order():
|
|
"""STRUCTURAL GUARD: the operator's terminal path must transition
|
|
BEFORE saving the cache (T21 21I fix). Greps the real source."""
|
|
src = (REPO / "src/rune/rune_operator.py").read_text()
|
|
# find the _apply_runtime_result block
|
|
idx = src.find("def _apply_runtime_result")
|
|
assert idx > 0
|
|
block = src[idx:idx + 6000]
|
|
save_at = block.find("self._save()")
|
|
trans_at = block.find('self._task_state.transition(task.task_id, "completed")')
|
|
assert trans_at > 0, "terminal transition missing"
|
|
if save_at > 0:
|
|
assert trans_at < save_at, (
|
|
"canonical transition must precede cache save in the terminal "
|
|
"path (21I crash-window fix regressed)")
|