Nick
c3c95688f7
Seeds the pipeline with the REAL #192 CONVERGED case-mismatch payload: the safety-net convergence step exported Python-cased True/False from json.load while the steps compared env.CONVERGED == 'false' — both conditionals unreachable (discharged on the release-safety-net branch, public in PR #211's diff). Captured via the pipeline itself (--class env-case-mismatch --source ci, 0 redactions — the payload is already public and credential-free). The generated regression test asserts the boolean-export checker still FLAGS the recorded payload (checker_expectation: flags), so a recurrence of the class fails CI. Manifest gains checker_expectation: capture records whether the class's checker raises on the payload ('flags' — the raise IS the recurrence alarm) or must stay quiet ('passes'); the generated tests enforce the recorded expectation, so a checker that stops catching its recorded defect rots the suite red. Docs: bugfix-contract §10 gains the operational capture step + explicit trivial-bug exemption list; README dev section points at the pipeline; tests/fixtures/production/README.md documents the flow. Henxels: capture_failure.py required in scripts/; production fixtures governed (manifest.json + README.md required); payload.json and manifest.json travel together (changed_with both directions, verified to bite on a payload-only staged change).
200 lines
9.3 KiB
Python
200 lines
9.3 KiB
Python
"""Tests for the manifest-driven regression test generator (#157, Task 2).
|
|
|
|
scripts/capture_failure.py embeds the canonical template for
|
|
``tests/test_production_fixtures.py``: one parametrized regression test per
|
|
manifest fixture asserting the system's CURRENT handling of the historical
|
|
payload — sha256 tamper-evidence first (a payload edited without re-running
|
|
capture must FAIL), sanitiser idempotence, metadata shape, and per-class
|
|
checker assertions (the recorded payload must be CAUGHT by the checker for
|
|
its failure class, so a recurrence fails CI).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def _load_module():
|
|
spec = importlib.util.spec_from_file_location("capture_failure", ROOT / "scripts" / "capture_failure.py")
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
cf = _load_module()
|
|
|
|
|
|
def _capture(
|
|
tmp_path: Path,
|
|
payload: str,
|
|
failure_class: str = "ci-crash",
|
|
source: str = "manual",
|
|
fixture_id: str | None = None,
|
|
):
|
|
payload_file = tmp_path / "payload.txt"
|
|
payload_file.write_text(payload, encoding="utf-8")
|
|
return cf.capture(
|
|
payload_file,
|
|
failure_class=failure_class,
|
|
source=source,
|
|
fixture_id=fixture_id,
|
|
fixtures_root=tmp_path / "fixtures",
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Generation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_capture_generates_the_regression_test_file(tmp_path: Path) -> None:
|
|
_capture(tmp_path, "boom once", fixture_id="gen-case")
|
|
generated = tmp_path / "fixtures" / "test_production_fixtures.py"
|
|
assert generated.exists(), "capture must generate the manifest-driven regression test file"
|
|
source = generated.read_text(encoding="utf-8")
|
|
assert "manifest.json" in source, "the generated test file must reference the manifest"
|
|
assert "sha256" in source, "the generated test file must assert payload sha256 (tamper-evidence)"
|
|
|
|
|
|
def test_generation_is_idempotent_and_never_clobbers(tmp_path: Path) -> None:
|
|
_capture(tmp_path, "boom once", fixture_id="idem-case")
|
|
generated = tmp_path / "fixtures" / "test_production_fixtures.py"
|
|
before = generated.read_text(encoding="utf-8")
|
|
_capture(tmp_path, "boom twice", fixture_id="idem-case-2")
|
|
assert generated.read_text(encoding="utf-8") == before, "regeneration must be byte-identical (no churn)"
|
|
|
|
|
|
def test_generated_test_file_passes_on_well_formed_fixture(tmp_path: Path) -> None:
|
|
_capture(tmp_path, "boom once", fixture_id="good-case")
|
|
generated = tmp_path / "fixtures" / "test_production_fixtures.py"
|
|
spec = importlib.util.spec_from_file_location("gen_fixtures", generated)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
manifest = json.loads((tmp_path / "fixtures" / "manifest.json").read_text(encoding="utf-8"))
|
|
for entry in manifest["fixtures"]:
|
|
module.assert_fixture_ok(entry, tmp_path / "fixtures") # must not raise
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The kill test: tampered payload FAILS the sha assertion
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_tampered_payload_fails_the_generated_assertion(tmp_path: Path) -> None:
|
|
_capture(tmp_path, "boom once", fixture_id="tamper-case")
|
|
generated = tmp_path / "fixtures" / "test_production_fixtures.py"
|
|
spec = importlib.util.spec_from_file_location("gen_fixtures_tamper", generated)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
|
|
fixture_file = tmp_path / "fixtures" / "ci-crash" / "tamper-case" / "payload.json"
|
|
stored = json.loads(fixture_file.read_text(encoding="utf-8"))
|
|
stored["payload"] = stored["payload"] + " TAMPERED" # edited without re-running capture
|
|
fixture_file.write_text(json.dumps(stored), encoding="utf-8")
|
|
|
|
manifest = json.loads((tmp_path / "fixtures" / "manifest.json").read_text(encoding="utf-8"))
|
|
entry = next(e for e in manifest["fixtures"] if e["id"] == "tamper-case")
|
|
with pytest.raises(AssertionError) as excinfo:
|
|
module.assert_fixture_ok(entry, tmp_path / "fixtures")
|
|
assert "sha256" in str(excinfo.value), "tamper must be reported as a sha mismatch"
|
|
|
|
|
|
def test_sha_detects_any_payload_byte_change(tmp_path: Path) -> None:
|
|
raw = "boom once"
|
|
_capture(tmp_path, raw, fixture_id="sha-case")
|
|
manifest = json.loads((tmp_path / "fixtures" / "manifest.json").read_text(encoding="utf-8"))
|
|
entry = next(e for e in manifest["fixtures"] if e["id"] == "sha-case")
|
|
assert entry["sha256"] == hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
|
assert entry["sha256"] != hashlib.sha256((raw + "\n").encode("utf-8")).hexdigest()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-class checkers: the payload must be CAUGHT by its class's checker
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_env_case_mismatch_checker_flags_the_defect(tmp_path: Path) -> None:
|
|
"""The #192 CONVERGED payload: a boolean env export emitting Python-cased
|
|
True/False while the consumer compares lowercase — the checker for this
|
|
class must FLAG the recorded payload (recurrence fails CI)."""
|
|
broken = "CONVERGED=$(printf '%s' \"$DECISION\" | python3 -c 'import json,sys; print(json.load(sys.stdin)[\"converged\"])')"
|
|
with pytest.raises(AssertionError) as excinfo:
|
|
cf.check_payload("env-case-mismatch", broken)
|
|
assert "lowercase" in str(excinfo.value)
|
|
|
|
|
|
def test_env_case_mismatch_checker_passes_the_fixed_form() -> None:
|
|
fixed = "CONVERGED=$(printf '%s' \"$DECISION\" | python3 -c 'import json,sys; print(str(json.load(sys.stdin)[\"converged\"]).lower())')"
|
|
cf.check_payload("env-case-mismatch", fixed) # must not raise
|
|
|
|
|
|
def test_env_case_mismatch_checker_ignores_non_export_lines() -> None:
|
|
"""The #192 payload's consumer line (`if: env.CONVERGED == 'false'`) and
|
|
prose comments must not trip the checker on their own."""
|
|
cf.check_payload("env-case-mismatch", "if: env.CONVERGED == 'false' && env.WARRANTED_BUMP != ''")
|
|
cf.check_payload("env-case-mismatch", "# the export emitted Python's capitalized True/False")
|
|
|
|
|
|
def test_check_payload_unknown_class_is_noop() -> None:
|
|
cf.check_payload("some-future-class", "anything at all") # generic assertions only
|
|
|
|
|
|
def test_generated_test_file_wires_the_class_checker(tmp_path: Path) -> None:
|
|
_capture(tmp_path, "CONVERGED=broken", failure_class="env-case-mismatch", fixture_id="wired-case")
|
|
generated = tmp_path / "fixtures" / "test_production_fixtures.py"
|
|
source = generated.read_text(encoding="utf-8")
|
|
assert "check_payload" in source, "generated tests must dispatch to the per-class checker"
|
|
# known classes are dispatched from the capture module's CLASS_CHECKERS —
|
|
# the generated file delegates rather than hardcoding the class list
|
|
capture_source = (ROOT / "scripts" / "capture_failure.py").read_text(encoding="utf-8")
|
|
assert '"env-case-mismatch"' in capture_source
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Checker expectation: defective payloads must RAISE, clean ones must not.
|
|
# The manifest records the expectation so the generated regression test can
|
|
# assert the CHECKING itself: a defect-class fixture passes only when the
|
|
# current checker still catches the recorded payload (recurrence fails CI);
|
|
# a post-fix/shape fixture passes only when the checker stays quiet.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_capture_records_checker_expectation_flags_for_defect_class(tmp_path: Path) -> None:
|
|
entry = _capture(
|
|
tmp_path,
|
|
"CONVERGED=$(printf '%s' \"$DECISION\" | python3 -c 'import json,sys; print(json.load(sys.stdin)[\"converged\"])')",
|
|
failure_class="env-case-mismatch",
|
|
fixture_id="defective-case",
|
|
)
|
|
assert entry["checker_expectation"] == "flags", "a payload exhibiting the defect must be recorded as 'flags'"
|
|
|
|
|
|
def test_capture_records_checker_expectation_passes_for_clean_class(tmp_path: Path) -> None:
|
|
entry = _capture(tmp_path, "boom once", failure_class="ci-crash", fixture_id="clean-class-case")
|
|
assert entry["checker_expectation"] == "passes", "a class without a checker cannot flag anything"
|
|
|
|
|
|
def test_generated_test_asserts_flags_for_the_converged_fixture(tmp_path: Path) -> None:
|
|
"""End-to-end: an env-case-mismatch fixture with the broken export must
|
|
produce a PASSING regression test whose mechanism is the checker RAISING."""
|
|
entry = _capture(
|
|
tmp_path,
|
|
"CONVERGED=$(printf '%s' \"$DECISION\" | python3 -c 'import json,sys; print(json.load(sys.stdin)[\"converged\"])')",
|
|
failure_class="env-case-mismatch",
|
|
fixture_id="real-defect",
|
|
)
|
|
assert entry["checker_expectation"] == "flags"
|
|
# the assertion helper used by the generated tests accepts the expectation
|
|
generated = tmp_path / "fixtures" / "test_production_fixtures.py"
|
|
source = generated.read_text(encoding="utf-8")
|
|
assert "checker_expectation" in source, "generated tests must honour the recorded expectation"
|