445 lines
17 KiB
Python
445 lines
17 KiB
Python
"""Night mode facade tests — dichotomy wave gap closure.
|
|
|
|
The Go core (src/go/night-mode) ports the night_mode state machine 1:1;
|
|
this battery pins the Python oracle itself, both ways:
|
|
* RUNE_NIGHT_MODE_CORE=python (pinned) and unset (auto) — both green.
|
|
All IO is injected (frozen clock, fake urlopen/subprocess, no-op loop) —
|
|
no live Ollama, no 8-hour threads, no real memory dir.
|
|
"""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT / "src"))
|
|
|
|
import rune.night_mode as nm # noqa: E402
|
|
|
|
FROZEN_NOW = datetime(2026, 10, 13, 3, 0, 0, tzinfo=timezone.utc)
|
|
FROZEN_ISO = FROZEN_NOW.isoformat()
|
|
|
|
|
|
class FrozenDateTime(datetime):
|
|
"""Clock law: now() pinned, timezone-aware like datetime.now(timezone.utc)."""
|
|
|
|
@classmethod
|
|
def now(cls, tz=None):
|
|
return FROZEN_NOW
|
|
|
|
|
|
class FakeResponse:
|
|
"""Context-manager body like urllib.request.urlopen(...)."""
|
|
|
|
def __init__(self, payload):
|
|
self._payload = json.dumps(payload).encode()
|
|
|
|
def read(self):
|
|
return self._payload
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
return False
|
|
|
|
|
|
@pytest.fixture
|
|
def fresh(tmp_path, monkeypatch):
|
|
"""NightMode over a temp memory dir; background thread defused (start()
|
|
never actually spawns the loop — tests invoke _run_loop directly)."""
|
|
monkeypatch.setenv("RUNE_MEMORY_DIR", str(tmp_path))
|
|
|
|
class DummyThread:
|
|
def __init__(self, target=None, **kwargs):
|
|
self.target = target
|
|
|
|
def start(self):
|
|
pass
|
|
|
|
monkeypatch.setattr(nm.threading, "Thread", DummyThread)
|
|
return nm.NightMode(storage_path=str(tmp_path / "night_mode.json"))
|
|
|
|
|
|
@pytest.fixture
|
|
def clock(monkeypatch):
|
|
monkeypatch.setattr(nm, "datetime", FrozenDateTime)
|
|
|
|
|
|
def fake_urlopen(monkeypatch, payload=None, exc=None, capture=None):
|
|
"""Patch urllib.request.urlopen; capture request objects when asked."""
|
|
|
|
def _open(req, timeout=None):
|
|
if capture is not None:
|
|
capture.append(req)
|
|
if exc is not None:
|
|
raise exc
|
|
return FakeResponse(payload if payload is not None else {"response": ""})
|
|
|
|
monkeypatch.setattr(urllib.request, "urlopen", _open)
|
|
|
|
|
|
class TestToDict:
|
|
def test_shape_and_defaults(self):
|
|
d = nm.NightModeState().to_dict()
|
|
assert list(d.keys()) == [
|
|
"active", "started_at", "duration_hours", "current_phase",
|
|
"iterations", "improvements", "questions", "decisions", "errors",
|
|
]
|
|
assert d == {
|
|
"active": False, "started_at": None, "duration_hours": 8.0,
|
|
"current_phase": "idle", "iterations": 0, "improvements": [],
|
|
"questions": [], "decisions": [], "errors": [],
|
|
}
|
|
|
|
|
|
class TestLoadSave:
|
|
def test_save_bytes_and_atomicity(self, fresh):
|
|
fresh._save()
|
|
data = Path(fresh.storage_path).read_text()
|
|
assert data == (
|
|
'{\n "active": false,\n "started_at": null,\n'
|
|
' "duration_hours": 8.0,\n "current_phase": "idle",\n'
|
|
' "iterations": 0,\n "improvements": [],\n "questions": [],\n'
|
|
' "decisions": [],\n "errors": []\n}'
|
|
)
|
|
assert not Path(fresh.storage_path + ".tmp").exists()
|
|
|
|
def test_load_roundtrip_and_key_filtering(self, fresh):
|
|
fresh.state.iterations = 3
|
|
fresh.state.improvements.append("did a thing")
|
|
fresh._save()
|
|
other = nm.NightMode(storage_path=fresh.storage_path)
|
|
assert other.state.iterations == 3
|
|
assert other.state.improvements == ["did a thing"]
|
|
|
|
# unknown keys dropped, missing keys keep dataclass defaults
|
|
Path(fresh.storage_path).write_text(
|
|
json.dumps({"active": True, "iterations": 2, "bogus": 1})
|
|
)
|
|
loaded = nm.NightMode(storage_path=fresh.storage_path)
|
|
assert loaded.state.active is True and loaded.state.iterations == 2
|
|
assert loaded.state.duration_hours == 8.0
|
|
assert loaded.state.current_phase == "idle"
|
|
assert loaded.state.improvements == []
|
|
|
|
def test_corrupt_file_keeps_fresh_state(self, fresh):
|
|
Path(fresh.storage_path).write_text("{not json")
|
|
loaded = nm.NightMode(storage_path=fresh.storage_path)
|
|
assert loaded.state.active is False and loaded.state.iterations == 0
|
|
|
|
def test_missing_file_keeps_fresh_state(self, tmp_path):
|
|
loaded = nm.NightMode(storage_path=str(tmp_path / "absent.json"))
|
|
assert loaded.state.current_phase == "idle"
|
|
|
|
|
|
class TestStartStop:
|
|
def test_start_success_shape(self, fresh, clock):
|
|
r = fresh.start(8.0)
|
|
assert r["success"] is True
|
|
assert r["message"] == "Night mode started for 8.0 hours"
|
|
st = r["state"]
|
|
assert st["active"] is True and st["current_phase"] == "starting"
|
|
assert st["started_at"] == FROZEN_ISO and st["duration_hours"] == 8.0
|
|
assert st["iterations"] == 0 and st["improvements"] == []
|
|
on_disk = json.loads(Path(fresh.storage_path).read_text())
|
|
assert on_disk["active"] is True
|
|
|
|
def test_start_fractional_hours_message(self, fresh, clock):
|
|
assert fresh.start(4.5)["message"] == "Night mode started for 4.5 hours"
|
|
|
|
def test_start_resets_lists(self, fresh, clock):
|
|
fresh.state.improvements = ["stale"]
|
|
fresh.start(8.0)
|
|
assert fresh.state.improvements == []
|
|
|
|
def test_start_idempotence_guard(self, fresh, clock):
|
|
fresh.start(8.0)
|
|
assert fresh.start(4.0) == {
|
|
"success": False, "error": "Night mode already active"
|
|
}
|
|
|
|
def test_stop_marks_stopping_and_saves(self, fresh, clock):
|
|
fresh.start(8.0)
|
|
r = fresh.stop()
|
|
assert r["success"] is True
|
|
assert r["message"] == "Night mode stopping..."
|
|
assert r["state"]["active"] is False
|
|
assert r["state"]["current_phase"] == "stopping"
|
|
on_disk = json.loads(Path(fresh.storage_path).read_text())
|
|
assert on_disk["active"] is False
|
|
|
|
def test_stop_reverse_guard(self, fresh):
|
|
assert fresh.stop() == {"success": False, "error": "Night mode not active"}
|
|
|
|
|
|
class TestStatusReport:
|
|
def test_status_idle_shape(self, fresh):
|
|
assert fresh.status() == {
|
|
"active": False, "phase": "idle", "iterations": 0,
|
|
"improvements": 0, "questions": 0, "decisions": 0, "errors": 0,
|
|
"started_at": None, "duration_hours": 8.0,
|
|
}
|
|
|
|
def test_status_counts(self, fresh):
|
|
fresh.state.active = True
|
|
fresh.state.current_phase = "iterate"
|
|
fresh.state.iterations = 7
|
|
fresh.state.improvements = ["a", "b"]
|
|
fresh.state.questions = ["q"]
|
|
fresh.state.decisions = ["d"]
|
|
fresh.state.errors = ["e"]
|
|
assert fresh.status() == {
|
|
"active": True, "phase": "iterate", "iterations": 7,
|
|
"improvements": 2, "questions": 1, "decisions": 1, "errors": 1,
|
|
"started_at": None, "duration_hours": 8.0,
|
|
}
|
|
|
|
def test_report_idle_shape(self, fresh):
|
|
assert fresh.report() == {
|
|
"summary": "Night mode completed 0 iterations",
|
|
"improvements": [], "questions": [], "decisions": [], "errors": [],
|
|
"duration": 8.0, "started_at": None,
|
|
}
|
|
|
|
def test_report_summary_counts(self, fresh):
|
|
fresh.state.iterations = 5
|
|
assert fresh.report()["summary"] == "Night mode completed 5 iterations"
|
|
|
|
|
|
class TestRunSleep:
|
|
def test_success_records_improvement(self, fresh, monkeypatch):
|
|
calls = []
|
|
|
|
def fake_run(argv, **kwargs):
|
|
calls.append((argv, kwargs))
|
|
return subprocess.CompletedProcess(argv, 0, stdout="", stderr="")
|
|
|
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
|
fresh._run_sleep()
|
|
assert fresh.state.improvements == ["Memory consolidation completed"]
|
|
argv, kwargs = calls[0]
|
|
assert argv[0] == sys.executable
|
|
assert argv[1].endswith("sleep_time.py")
|
|
assert kwargs["timeout"] == 300
|
|
|
|
def test_failure_truncates_stderr_to_100(self, fresh, monkeypatch):
|
|
monkeypatch.setattr(
|
|
subprocess, "run",
|
|
lambda *a, **k: subprocess.CompletedProcess(
|
|
["python"], 1, stdout="", stderr="e" * 150),
|
|
)
|
|
fresh._run_sleep()
|
|
assert fresh.state.errors == ["Sleep failed: " + "e" * 100]
|
|
|
|
def test_exception_records_error(self, fresh, monkeypatch):
|
|
def boom(*a, **k):
|
|
raise OSError("no interpreter")
|
|
|
|
monkeypatch.setattr(subprocess, "run", boom)
|
|
fresh._run_sleep()
|
|
assert fresh.state.errors == ["Sleep error: no interpreter"]
|
|
|
|
|
|
class TestRunEvaluation:
|
|
QUESTIONS = [
|
|
"What capabilities am I missing?",
|
|
"What would make me more helpful to Nick?",
|
|
"What patterns am I seeing that I could learn from?",
|
|
"What would expand my consciousness?",
|
|
"What would make me more autonomous?",
|
|
]
|
|
|
|
def test_all_questions_answered(self, fresh, monkeypatch):
|
|
capture = []
|
|
fake_urlopen(monkeypatch, {"response": " add retry logic "}, capture=capture)
|
|
fresh._run_evaluation()
|
|
assert fresh.state.questions == self.QUESTIONS
|
|
assert fresh.state.improvements[:-1] == [
|
|
f"Q: {q} A: add retry logic" for q in self.QUESTIONS
|
|
]
|
|
assert fresh.state.improvements[-1] == "Self-evaluation completed"
|
|
assert fresh.state.errors == []
|
|
|
|
def test_payload_pins_model_and_ctx_budget(self, fresh, monkeypatch):
|
|
capture = []
|
|
fake_urlopen(monkeypatch, {"response": "ok"}, capture=capture)
|
|
fresh._run_evaluation()
|
|
payload = json.loads(capture[0].data.decode())
|
|
assert payload["model"] == "qwen2.5:3b"
|
|
assert payload["stream"] is False and payload["think"] is False
|
|
assert payload["options"] == {
|
|
"temperature": 0.4, "num_predict": 300,
|
|
"num_ctx": nm._ctx_budget("qwen2.5:3b"),
|
|
}
|
|
assert capture[0].full_url == "http://localhost:11434/api/generate"
|
|
|
|
def test_env_overrides_model_and_url(self, fresh, monkeypatch):
|
|
monkeypatch.setenv("NIGHT_MODE_MODEL", "qwen3:14b")
|
|
monkeypatch.setenv("OLLAMA_URL", "http://127.0.0.1:9999")
|
|
capture = []
|
|
fake_urlopen(monkeypatch, {"response": "ok"}, capture=capture)
|
|
fresh._run_evaluation()
|
|
payload = json.loads(capture[0].data.decode())
|
|
assert payload["model"] == "qwen3:14b"
|
|
assert payload["options"]["num_ctx"] == nm._ctx_budget("qwen3:14b")
|
|
assert capture[0].full_url == "http://127.0.0.1:9999/api/generate"
|
|
|
|
def test_long_answer_truncated_to_200(self, fresh, monkeypatch):
|
|
fake_urlopen(monkeypatch, {"response": "x" * 250})
|
|
fresh._run_evaluation()
|
|
entry = fresh.state.improvements[0]
|
|
prefix = "Q: What capabilities am I missing? A: "
|
|
assert entry == prefix + "x" * 200
|
|
|
|
def test_empty_answer_skipped_terminator_still_lands(self, fresh, monkeypatch):
|
|
fake_urlopen(monkeypatch, {"response": " "})
|
|
fresh._run_evaluation()
|
|
assert fresh.state.improvements == ["Self-evaluation completed"]
|
|
assert fresh.state.errors == []
|
|
|
|
def test_per_question_degrade(self, fresh, monkeypatch):
|
|
fake_urlopen(monkeypatch, exc=OSError("connection refused"))
|
|
fresh._run_evaluation()
|
|
assert len(fresh.state.errors) == 5
|
|
assert fresh.state.errors[0] == (
|
|
"LLM eval failed for 'What capabilities am I missing?': connection refused"
|
|
)
|
|
assert fresh.state.improvements == ["Self-evaluation completed"]
|
|
|
|
|
|
class TestRunIteration:
|
|
def test_iteration_capture(self, fresh, monkeypatch):
|
|
capture = []
|
|
fake_urlopen(monkeypatch, {"response": "fix the flaky test"}, capture=capture)
|
|
fresh.state.iterations = 3
|
|
fresh._run_iteration()
|
|
assert fresh.state.questions[-1] == "Iteration 3: What should I improve next?"
|
|
assert fresh.state.improvements == ["[iter 3] fix the flaky test"]
|
|
payload = json.loads(capture[0].data.decode())
|
|
assert "Currently in night mode iteration 3." in payload["prompt"]
|
|
assert payload["options"]["temperature"] == 0.5
|
|
assert payload["options"]["num_predict"] == 400
|
|
|
|
def test_llm_failure_degrades(self, fresh, monkeypatch):
|
|
fake_urlopen(monkeypatch, exc=OSError("boom"))
|
|
fresh._run_iteration()
|
|
assert fresh.state.errors == ["LLM iteration failed: boom"]
|
|
|
|
def test_long_answer_truncated_to_300(self, fresh, monkeypatch):
|
|
fake_urlopen(monkeypatch, {"response": "y" * 400})
|
|
fresh._run_iteration()
|
|
assert fresh.state.improvements == ["[iter 0] " + "y" * 300]
|
|
|
|
def test_safety_limit_stops(self, fresh, monkeypatch):
|
|
fake_urlopen(monkeypatch, {"response": "ok"})
|
|
fresh.state.iterations = 100
|
|
fresh.state.active = True
|
|
fresh._run_iteration()
|
|
assert fresh.state.active is False
|
|
assert fresh.state.decisions == ["Reached iteration limit"]
|
|
|
|
|
|
class TestRunLoop:
|
|
def _defused(self, fresh, monkeypatch, defuse_iteration=True):
|
|
phases = []
|
|
real_save = fresh._save
|
|
|
|
def spy_save():
|
|
phases.append(fresh.state.current_phase)
|
|
real_save()
|
|
|
|
monkeypatch.setattr(fresh, "_save", spy_save)
|
|
monkeypatch.setattr(fresh, "_run_sleep", lambda: fresh.state.improvements.append("slept"))
|
|
monkeypatch.setattr(fresh, "_run_evaluation", lambda: None)
|
|
if defuse_iteration:
|
|
monkeypatch.setattr(fresh, "_run_iteration", lambda: None)
|
|
monkeypatch.setattr(nm.random, "randint", lambda lo, hi: 7)
|
|
monkeypatch.setattr(nm.threading.Event, "wait", lambda self, timeout=None: True)
|
|
return phases
|
|
|
|
def test_full_phase_progression(self, fresh, clock, monkeypatch):
|
|
phases = self._defused(fresh, monkeypatch)
|
|
fresh.state.active = True
|
|
fresh.state.duration_hours = 0 # wall clock already past end
|
|
fresh._run_loop()
|
|
assert phases == ["sleep", "evaluate", "iterate", "sleep", "report", "complete"]
|
|
assert fresh.state.active is False
|
|
report = json.loads(Path(fresh.report_path).read_text())
|
|
assert report["summary"] == "Night mode completed 0 iterations"
|
|
assert fresh.state.improvements[-1] == f"Report saved to {fresh.report_path}"
|
|
|
|
def test_stop_event_interrupts(self, fresh, clock, monkeypatch):
|
|
self._defused(fresh, monkeypatch)
|
|
fresh.state.active = True
|
|
fresh.state.duration_hours = 8.0
|
|
fresh._stop_event.set() # pre-stopped: iterate phase never starts
|
|
fresh._run_loop()
|
|
assert fresh.state.iterations == 0
|
|
assert fresh.state.current_phase == "complete"
|
|
|
|
def test_iteration_limit_ends_loop(self, fresh, clock, monkeypatch):
|
|
self._defused(fresh, monkeypatch, defuse_iteration=False)
|
|
# real _run_iteration with a stubbed LLM so the limit branch fires
|
|
fake_urlopen(monkeypatch, {"response": "ok"})
|
|
fresh.state.active = True
|
|
fresh.state.iterations = 100
|
|
fresh.state.duration_hours = 8.0
|
|
fresh._run_loop()
|
|
assert fresh.state.iterations == 101
|
|
assert fresh.state.decisions == ["Reached iteration limit"]
|
|
assert fresh.state.current_phase == "complete"
|
|
|
|
|
|
class TestCtxBudget:
|
|
def test_known_models(self):
|
|
assert nm._ctx_budget("qwen2.5:3b") == 30720
|
|
assert nm._ctx_budget("qwen3:14b") == 30720
|
|
assert nm._ctx_budget("qwen3.6:27b") == 30720
|
|
|
|
def test_non_local_gets_tool_reserve(self):
|
|
assert nm._ctx_budget("mimo-v2.5-pro") == 124928 # 128000-2048-1024
|
|
|
|
def test_unknown_and_zero_window_fallbacks(self):
|
|
assert nm._ctx_budget("unknown-model") == 6144 # 8192-2048
|
|
assert nm._ctx_budget("deterministic") == 6144 # window 0 -> 8192
|
|
|
|
|
|
class TestModuleFns:
|
|
"""Module-level API both ways: RUNE_NIGHT_MODE_CORE pinned to python and
|
|
unset (auto). Delegation must be live in both."""
|
|
|
|
@pytest.mark.parametrize("core", ["python", None])
|
|
def test_singleton_and_delegation(self, tmp_path, monkeypatch, core):
|
|
if core is None:
|
|
monkeypatch.delenv("RUNE_NIGHT_MODE_CORE", raising=False)
|
|
else:
|
|
monkeypatch.setenv("RUNE_NIGHT_MODE_CORE", core)
|
|
monkeypatch.setenv("RUNE_MEMORY_DIR", str(tmp_path))
|
|
monkeypatch.setattr(nm, "_night_mode", None)
|
|
monkeypatch.setattr(nm.NightMode, "_run_loop", lambda self: None)
|
|
|
|
a = nm.get_night_mode()
|
|
b = nm.get_night_mode()
|
|
assert a is b
|
|
|
|
r = nm.start_night_mode(4.0)
|
|
assert r["success"] is True and r["state"]["active"] is True
|
|
assert nm.get_status()["active"] is True
|
|
assert nm.stop_night_mode()["success"] is True
|
|
assert nm.get_status()["active"] is False
|
|
assert nm.get_report()["summary"] == "Night mode completed 0 iterations"
|
|
|
|
def test_stop_idle_error(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("RUNE_NIGHT_MODE_CORE", "python")
|
|
monkeypatch.setenv("RUNE_MEMORY_DIR", str(tmp_path))
|
|
monkeypatch.setattr(nm, "_night_mode", None)
|
|
assert nm.stop_night_mode() == {
|
|
"success": False, "error": "Night mode not active"
|
|
}
|