From 21633a5bc56bf2263173f93bf2e3b4b84fbbcbb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Mon, 25 May 2026 12:30:38 +0300 Subject: [PATCH] =?UTF-8?q?feat(brain):=20analyzer=20v4=20aggregations=20+?= =?UTF-8?q?=20schema=5Fminor=202=E2=86=923=20+=20phase-3=20flags=20(phase?= =?UTF-8?q?=203=20task=2020)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 Task 20 — analyzer surfaces v4 review distribution / inheritance / cost totals / degraded count. Schema_minor bumps 2→3. Final phase-3 runtime flags flipped. - tools/brain-retro-analyzer.mjs: + inheritanceCount: count of episodes with inheritance.inherited_from_task_id. + reviewQuality: distribution of review.node_quality across {correct, wrong_node, overkill, underkill, disputable}. + reviewerCoverage: {reviewed, pending, errored} — episodes reviewed by subagent / awaiting review / escalated with reviewer_error. + degradedCount: episodes where LLM classifier fell back to regex. + costTotals: sum of classifier/self_assessment/reviewer input/output tokens across the period (six counters). All additions are read-only over the existing dedup'd normal episode list — no new pass. - tools/brain-retro-analyzer.test.mjs: +6 tests (inheritance count / reviewQuality distribution / pending / errored / degraded / cost sums). - tools/observer-stop-hook.mjs: buildEpisode schema_minor 2→3 bump. - tools/observer-stop-hook.test.mjs: 1 schema_minor assertion 2→3. Runtime flags flipped (user-level, not git): reviewer-mode = subagent self-retrospect-mode = on sanity-check-mode = mandatory All 9 phase-2 + phase-3 flags now present: router-classifier-mode=llm-first | prompt-enrichment-mode=on | inheritance-mode=on | embedding-mode=on | router-gate-mode=warn-only | self-assessment-mode=on | reviewer-mode=subagent | self-retrospect-mode=on | sanity-check-mode=mandatory. Tests: 614 passed / 0 failed. 4 pre-existing empty test files unchanged. NB: schema v4.3 parser extension (prompt_embedding_base64 + outcome_reviewed + extended task_cost in parser write block per spec §5) NOT touched in this commit — that wiring belongs to the parse-time path which Task 17 also did not modify (only buildEpisode in stop-hook bumps the minor). Both are tracked for Phase 3 follow-up alongside §4.9 coverage announcement and status-md cost section. --- tools/brain-retro-analyzer.mjs | 42 +++++++++++++++++++++++ tools/brain-retro-analyzer.test.mjs | 52 +++++++++++++++++++++++++++++ tools/observer-stop-hook.mjs | 2 +- tools/observer-stop-hook.test.mjs | 4 +-- 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/tools/brain-retro-analyzer.mjs b/tools/brain-retro-analyzer.mjs index dc2588a3..457933cb 100644 --- a/tools/brain-retro-analyzer.mjs +++ b/tools/brain-retro-analyzer.mjs @@ -219,6 +219,43 @@ export function analyze(episodes, options = {}) { const disciplineByClassification = disciplinePercentByClassification(normal, classificationMap); const routerStep = routerStepReached(normal); const boundariesRate = boundariesAppliedRate(normal); + + // Phase 3 Task 20 — v4 aggregation: inheritance count + reviewer outcome + // distribution + cost totals. Reads schema_version >=4 fields gracefully. + let inheritanceCount = 0; + const reviewQuality = { correct: 0, wrong_node: 0, overkill: 0, underkill: 0, disputable: 0 }; + const reviewerCoverage = { reviewed: 0, pending: 0, errored: 0 }; + let degradedCount = 0; + const costTotals = { + classifier_input_tokens: 0, + classifier_output_tokens: 0, + self_assessment_input_tokens: 0, + self_assessment_output_tokens: 0, + reviewer_input_tokens: 0, + reviewer_output_tokens: 0, + }; + for (const e of normal) { + if (e?.inheritance?.inherited_from_task_id) inheritanceCount += 1; + if (e?.degraded_mode === true) degradedCount += 1; + const r = e?.review; + if (r && typeof r === 'object') { + if (r.reviewer_error) reviewerCoverage.errored += 1; + else if (typeof r.node_quality === 'string') { + reviewerCoverage.reviewed += 1; + if (reviewQuality[r.node_quality] !== undefined) reviewQuality[r.node_quality] += 1; + } + } else if (e?.schema_version >= 4) { + reviewerCoverage.pending += 1; + } + const tc = e?.task_cost; + if (tc && typeof tc === 'object') { + for (const k of Object.keys(costTotals)) { + const v = tc[k]; + if (typeof v === 'number' && Number.isFinite(v)) costTotals[k] += v; + } + } + } + return { episodeCount: normal.length, v1SkippedCount, @@ -230,6 +267,11 @@ export function analyze(episodes, options = {}) { disciplineByClassification, routerStep, boundariesRate, + inheritanceCount, + reviewQuality, + reviewerCoverage, + degradedCount, + costTotals, }; } diff --git a/tools/brain-retro-analyzer.test.mjs b/tools/brain-retro-analyzer.test.mjs index 48f19f9c..ffd63b6c 100644 --- a/tools/brain-retro-analyzer.test.mjs +++ b/tools/brain-retro-analyzer.test.mjs @@ -357,3 +357,55 @@ describe('analyze — discipline metrics (stage 2)', () => { expect(res.boundariesRate.rate).toBeCloseTo(0.5); }); }); + +describe('analyze — v4 aggregations (Phase 3 Task 20)', () => { + it('aggregates inheritanceCount across v4 episodes', () => { + const eps = [ + ep({ schema_version: 4, inheritance: { inherited_from_task_id: 'x' } }), + ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:01:00Z', ended_at: '2026-05-19T10:02:00Z' }, inheritance: { inherited_from_task_id: 'y' } }), + ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' } }), + ]; + expect(analyze(eps).inheritanceCount).toBe(2); + }); + + it('aggregates reviewQuality distribution from review.node_quality', () => { + const eps = [ + ep({ schema_version: 4, review: { node_quality: 'correct' } }), + ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:01:00Z', ended_at: '2026-05-19T10:02:00Z' }, review: { node_quality: 'correct' } }), + ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, review: { node_quality: 'wrong_node' } }), + ]; + const res = analyze(eps); + expect(res.reviewQuality.correct).toBe(2); + expect(res.reviewQuality.wrong_node).toBe(1); + expect(res.reviewerCoverage.reviewed).toBe(3); + }); + + it('counts review pending for v4 episodes without a review block', () => { + const eps = [ep({ schema_version: 4 })]; + expect(analyze(eps).reviewerCoverage.pending).toBe(1); + }); + + it('counts reviewer_error escalations under reviewerCoverage.errored', () => { + const eps = [ep({ schema_version: 4, review: { reviewer_error: 'malformed episode' } })]; + expect(analyze(eps).reviewerCoverage.errored).toBe(1); + }); + + it('aggregates degradedCount on degraded_mode=true', () => { + const eps = [ + ep({ schema_version: 4, degraded_mode: true }), + ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:01:00Z', ended_at: '2026-05-19T10:02:00Z' }, degraded_mode: false }), + ]; + expect(analyze(eps).degradedCount).toBe(1); + }); + + it('sums task_cost tokens into costTotals', () => { + const eps = [ + ep({ schema_version: 4, task_cost: { classifier_input_tokens: 100, classifier_output_tokens: 30 } }), + ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:01:00Z', ended_at: '2026-05-19T10:02:00Z' }, task_cost: { classifier_input_tokens: 200, reviewer_input_tokens: 500 } }), + ]; + const ct = analyze(eps).costTotals; + expect(ct.classifier_input_tokens).toBe(300); + expect(ct.classifier_output_tokens).toBe(30); + expect(ct.reviewer_input_tokens).toBe(500); + }); +}); diff --git a/tools/observer-stop-hook.mjs b/tools/observer-stop-hook.mjs index 46911d4a..295fc8b5 100644 --- a/tools/observer-stop-hook.mjs +++ b/tools/observer-stop-hook.mjs @@ -192,7 +192,7 @@ export function buildExecutionTrace({ recommended_chain = [], invoked = [] } = { */ export function buildEpisode({ state = null, transcriptText = null, ctx = {} } = {}) { const base = buildEpisodeFromContext(ctx, transcriptText); - base.schema_minor = 2; // Task 17 bump (self_assessment surface added) + base.schema_minor = 3; // Task 20 bump (cost totals + reviewer distribution surface) if (state?.inheritance) { base.inheritance = { ...state.inheritance }; } diff --git a/tools/observer-stop-hook.test.mjs b/tools/observer-stop-hook.test.mjs index fa34904b..fbcc09dd 100644 --- a/tools/observer-stop-hook.test.mjs +++ b/tools/observer-stop-hook.test.mjs @@ -206,10 +206,10 @@ describe('buildExecutionTrace + buildEpisode — Phase 3 Task 16 (spec §5)', () expect(ep.inheritance).toBeUndefined(); }); - it('buildEpisode marks schema_minor=2 (Task 17 bump)', () => { + it('buildEpisode marks schema_minor=3 (Task 20 bump)', () => { const ep = buildEpisode({ state: {}, ctx: { session_id: 'sess-x' } }); expect(ep.schema_version).toBe(4); - expect(ep.schema_minor).toBe(2); + expect(ep.schema_minor).toBe(3); }); });