45 lines
1.2 KiB
Python
45 lines
1.2 KiB
Python
"""Benchmark harness and regression rule."""
|
|
|
|
from rune.orchestration.benchmark import (
|
|
evaluate_p95_regression,
|
|
fingerprint,
|
|
run_batches,
|
|
)
|
|
|
|
|
|
def test_fingerprint_is_a_nonempty_dict():
|
|
fp = fingerprint()
|
|
assert "python" in fp
|
|
assert "platform" in fp
|
|
|
|
|
|
def test_run_batches_excludes_warmup():
|
|
calls: list[int] = []
|
|
|
|
def scenario(batch: int) -> float:
|
|
calls.append(batch)
|
|
return 0.001
|
|
|
|
batches = run_batches(scenario, batches=2, warmup=3, ops=2)
|
|
assert len(batches) == 2
|
|
assert [len(b) for b in batches] == [2, 2]
|
|
assert len(calls) == 2 * (3 + 2)
|
|
|
|
|
|
def test_no_regression_below_noise_and_threshold():
|
|
base = [0.010] * 100
|
|
cand = [0.010 + 1e-6] * 100 # tiny, below noise and 5%
|
|
assert evaluate_p95_regression(base, cand) is False
|
|
|
|
|
|
def test_regression_detected_at_ten_percent():
|
|
base = [0.010] * 100
|
|
cand = [0.0110001] * 100 # 10.001% slower: 1.0001 ms delta clears the 1 ms floor
|
|
assert evaluate_p95_regression(base, cand) is True
|
|
|
|
|
|
def test_regression_rule_uses_medians_not_means():
|
|
base = [0.010] * 99 + [0.5] # one huge outlier in baseline
|
|
cand = [0.0110001] * 100
|
|
assert evaluate_p95_regression(base, cand) is True
|