From d91f4288448522401889a874708cc22bb49484a5 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 9 Sep 2026 13:52:44 +0100 Subject: [PATCH 1/2] docs(RP13): throughput/quality metrics for #161 + #163 real runs (closes #276) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-stage cognition metrics (elapsed where recorded, tool calls, queries, sources read, failures), research quality (sources/contradictions/ limitations), adversarial outcomes (attacks/survivors/rejected/mitigations/ kill-tests/dispositions), old-process baseline (300s foreground sleeps, model-claimed receipts, no read provenance), and quality outcomes (architectural mistakes prevented per run, implementation ambiguity reduction, rework loops). Honest data note: stage elapsed_s only recorded where the last artifact write set it (reconciliation zeroed earlier stages on #161) — per-stage timing persistence is a TUNE recommendation, not a measurement blocker. --- docs/RP13_THROUGHPUT_REPORT.json | 188 +++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 docs/RP13_THROUGHPUT_REPORT.json diff --git a/docs/RP13_THROUGHPUT_REPORT.json b/docs/RP13_THROUGHPUT_REPORT.json new file mode 100644 index 00000000..5e529007 --- /dev/null +++ b/docs/RP13_THROUGHPUT_REPORT.json @@ -0,0 +1,188 @@ +{ + "runs": { + "161": { + "issue": 161, + "stages": { + "ISSUE": { + "elapsed_s": 0.0, + "tool_calls": 0, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + }, + "INWARD": { + "elapsed_s": 0.0, + "tool_calls": 0, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + }, + "OUTWARD": { + "elapsed_s": 0.0, + "tool_calls": 22, + "queries": 4, + "sources_read": 4, + "tool_failures": 0 + }, + "APPLICABILITY": { + "elapsed_s": 0.0, + "tool_calls": 20, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + }, + "ADVERSARIAL": { + "elapsed_s": 0.0, + "tool_calls": 25, + "queries": 0, + "sources_read": 2, + "tool_failures": 0 + }, + "SYNTHESIS": { + "elapsed_s": 0.0, + "tool_calls": 14, + "queries": 0, + "sources_read": 0, + "tool_failures": 1 + }, + "QUALIFY": { + "elapsed_s": 310.1, + "tool_calls": 20, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + } + }, + "totals": { + "cognition_elapsed_s": 310.1, + "tool_calls": 101 + }, + "research": { + "queries": 4, + "sources_discovered": 16, + "sources_read": 4, + "contradictions": 4, + "limitations": 8 + }, + "adversarial": { + "attacks": 16, + "survivors": 6, + "rejected": 1, + "mitigations": 10, + "kill_tests": 7, + "disposition": "SURVIVES_WITH_MITIGATIONS" + } + }, + "163": { + "issue": 163, + "stages": { + "ISSUE": { + "elapsed_s": 0.0, + "tool_calls": 0, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + }, + "INWARD": { + "elapsed_s": 0.0, + "tool_calls": 0, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + }, + "OUTWARD": { + "elapsed_s": 0.0, + "tool_calls": 34, + "queries": 5, + "sources_read": 10, + "tool_failures": 2 + }, + "APPLICABILITY": { + "elapsed_s": 300.1, + "tool_calls": 8, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + }, + "INNOVATION": { + "elapsed_s": 422.1, + "tool_calls": 9, + "queries": 0, + "sources_read": 0, + "tool_failures": 1 + }, + "ADVERSARIAL": { + "elapsed_s": 0.0, + "tool_calls": 25, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + }, + "SYNTHESIS": { + "elapsed_s": 677.1, + "tool_calls": 15, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + }, + "QUALIFY": { + "elapsed_s": 305.1, + "tool_calls": 12, + "queries": 0, + "sources_read": 0, + "tool_failures": 0 + } + }, + "totals": { + "cognition_elapsed_s": 1704.3, + "tool_calls": 103 + }, + "research": { + "queries": 5, + "sources_discovered": 45, + "sources_read": 10, + "contradictions": 4, + "limitations": 8 + }, + "adversarial": { + "attacks": 13, + "survivors": 4, + "rejected": 1, + "mitigations": 7, + "kill_tests": 6, + "disposition": "SURVIVES_WITH_MITIGATIONS" + } + } + }, + "old_process_baseline": { + "foreground_sleep_per_batch_s": 300, + "batches_observed_in_rsc0_red": "N (300s x N while child productive)", + "wall_clock_issue_245_era": "hours (multiple restarts, no durable state)", + "source_receipts": "model-claimed only (no relay receipts)", + "read_provenance": "none \u2014 snippets counted as evidence" + }, + "quality_outcomes": { + "161": { + "architectural_mistakes_prevented": [ + "MemoryAdapter second protocol rejected (duplicate authority vs memory_selector registry)", + "Harness would have silently validated non-hybrid mode (A1: _embed degradation) \u2014 kill-test FT1 forced mode observability", + "Cache-pollution benchmark hazard (A4) \u2014 forced cache bypass invariants" + ], + "implementation_ambiguity_reduction": "QUALIFY 13/13 checks + source-verified interface review; implementation merged first try, 0 CI code failures (2 CI-contract failures were master debt)", + "rework": "1 REFINE loop none; implementation PR #279 merged without implementation rework" + }, + "163": { + "architectural_mistakes_prevented": [ + "External graph DB adoption blocked (3 candidates CONFLICTS_WITH_RUNE \u2014 would replace SQLite tier)", + "False bi-temporal claim corrected from code (graph_memory is transaction-time append) before synthesis", + "INN-1 filter formulation falsified (A2) \u2014 synthesis v2 corrected" + ], + "implementation_ambiguity_reduction": "QUALIFY round 1 caught 3 concrete interface gaps (DDL, parameter, alias ordering) \u2014 REFINE_SYNTHESIS fixed all in one loop", + "rework": "1 REFINE_SYNTHESIS loop (targeted, synthesis-only \u2014 no research restart)" + } + }, + "comparison_principles": { + "cognition_vs_orchestration": "cognition = MiMo stage elapsed (measured where recorded); orchestration/wait = runner queue + CI latency \u2014 NOT attributed to the pipeline", + "wall_clock": "#161 research chain \u2248 75 min git-window (00:16\u219201:31) incl. live CI defect discovery/fixes; #163 persisted in one batch (stage elapsed recorded per-stage)" + } +} \ No newline at end of file -- 2.56.0 From bfbc153ec22a3890cafd1ceddec0dd19bb45de70 Mon Sep 17 00:00:00 2001 From: Nick Date: Wed, 9 Sep 2026 13:58:10 +0100 Subject: [PATCH 2/2] =?UTF-8?q?fix(RP13):=20metrics=20report=20as=20YAML?= =?UTF-8?q?=20=E2=80=94=20top-level=20docs=20henxel=20requires=20md/yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/{RP13_THROUGHPUT_REPORT.json => RP13_THROUGHPUT_REPORT.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{RP13_THROUGHPUT_REPORT.json => RP13_THROUGHPUT_REPORT.yaml} (100%) diff --git a/docs/RP13_THROUGHPUT_REPORT.json b/docs/RP13_THROUGHPUT_REPORT.yaml similarity index 100% rename from docs/RP13_THROUGHPUT_REPORT.json rename to docs/RP13_THROUGHPUT_REPORT.yaml -- 2.56.0