feat(brain): analyzer v4 aggregations + schema_minor 2→3 + phase-3 flags (phase 3 task 20)
Phase 3 Task 20 — analyzer surfaces v4 review distribution / inheritance /
cost totals / degraded count. Schema_minor bumps 2→3. Final phase-3 runtime
flags flipped.
- tools/brain-retro-analyzer.mjs:
+ inheritanceCount: count of episodes with inheritance.inherited_from_task_id.
+ reviewQuality: distribution of review.node_quality across
{correct, wrong_node, overkill, underkill, disputable}.
+ reviewerCoverage: {reviewed, pending, errored} — episodes reviewed by
subagent / awaiting review / escalated with reviewer_error.
+ degradedCount: episodes where LLM classifier fell back to regex.
+ costTotals: sum of classifier/self_assessment/reviewer input/output
tokens across the period (six counters).
All additions are read-only over the existing dedup'd normal episode
list — no new pass.
- tools/brain-retro-analyzer.test.mjs: +6 tests (inheritance count /
reviewQuality distribution / pending / errored / degraded / cost sums).
- tools/observer-stop-hook.mjs: buildEpisode schema_minor 2→3 bump.
- tools/observer-stop-hook.test.mjs: 1 schema_minor assertion 2→3.
Runtime flags flipped (user-level, not git):
reviewer-mode = subagent
self-retrospect-mode = on
sanity-check-mode = mandatory
All 9 phase-2 + phase-3 flags now present:
router-classifier-mode=llm-first | prompt-enrichment-mode=on |
inheritance-mode=on | embedding-mode=on | router-gate-mode=warn-only |
self-assessment-mode=on | reviewer-mode=subagent |
self-retrospect-mode=on | sanity-check-mode=mandatory.
Tests: 614 passed / 0 failed. 4 pre-existing empty test files unchanged.
NB: schema v4.3 parser extension (prompt_embedding_base64 +
outcome_reviewed + extended task_cost in parser write block per spec §5)
NOT touched in this commit — that wiring belongs to the parse-time path
which Task 17 also did not modify (only buildEpisode in stop-hook bumps
the minor). Both are tracked for Phase 3 follow-up alongside §4.9
coverage announcement and status-md cost section.
This commit is contained in:
@@ -219,6 +219,43 @@ export function analyze(episodes, options = {}) {
|
||||
const disciplineByClassification = disciplinePercentByClassification(normal, classificationMap);
|
||||
const routerStep = routerStepReached(normal);
|
||||
const boundariesRate = boundariesAppliedRate(normal);
|
||||
|
||||
// Phase 3 Task 20 — v4 aggregation: inheritance count + reviewer outcome
|
||||
// distribution + cost totals. Reads schema_version >=4 fields gracefully.
|
||||
let inheritanceCount = 0;
|
||||
const reviewQuality = { correct: 0, wrong_node: 0, overkill: 0, underkill: 0, disputable: 0 };
|
||||
const reviewerCoverage = { reviewed: 0, pending: 0, errored: 0 };
|
||||
let degradedCount = 0;
|
||||
const costTotals = {
|
||||
classifier_input_tokens: 0,
|
||||
classifier_output_tokens: 0,
|
||||
self_assessment_input_tokens: 0,
|
||||
self_assessment_output_tokens: 0,
|
||||
reviewer_input_tokens: 0,
|
||||
reviewer_output_tokens: 0,
|
||||
};
|
||||
for (const e of normal) {
|
||||
if (e?.inheritance?.inherited_from_task_id) inheritanceCount += 1;
|
||||
if (e?.degraded_mode === true) degradedCount += 1;
|
||||
const r = e?.review;
|
||||
if (r && typeof r === 'object') {
|
||||
if (r.reviewer_error) reviewerCoverage.errored += 1;
|
||||
else if (typeof r.node_quality === 'string') {
|
||||
reviewerCoverage.reviewed += 1;
|
||||
if (reviewQuality[r.node_quality] !== undefined) reviewQuality[r.node_quality] += 1;
|
||||
}
|
||||
} else if (e?.schema_version >= 4) {
|
||||
reviewerCoverage.pending += 1;
|
||||
}
|
||||
const tc = e?.task_cost;
|
||||
if (tc && typeof tc === 'object') {
|
||||
for (const k of Object.keys(costTotals)) {
|
||||
const v = tc[k];
|
||||
if (typeof v === 'number' && Number.isFinite(v)) costTotals[k] += v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
episodeCount: normal.length,
|
||||
v1SkippedCount,
|
||||
@@ -230,6 +267,11 @@ export function analyze(episodes, options = {}) {
|
||||
disciplineByClassification,
|
||||
routerStep,
|
||||
boundariesRate,
|
||||
inheritanceCount,
|
||||
reviewQuality,
|
||||
reviewerCoverage,
|
||||
degradedCount,
|
||||
costTotals,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -357,3 +357,55 @@ describe('analyze — discipline metrics (stage 2)', () => {
|
||||
expect(res.boundariesRate.rate).toBeCloseTo(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyze — v4 aggregations (Phase 3 Task 20)', () => {
|
||||
it('aggregates inheritanceCount across v4 episodes', () => {
|
||||
const eps = [
|
||||
ep({ schema_version: 4, inheritance: { inherited_from_task_id: 'x' } }),
|
||||
ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:01:00Z', ended_at: '2026-05-19T10:02:00Z' }, inheritance: { inherited_from_task_id: 'y' } }),
|
||||
ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' } }),
|
||||
];
|
||||
expect(analyze(eps).inheritanceCount).toBe(2);
|
||||
});
|
||||
|
||||
it('aggregates reviewQuality distribution from review.node_quality', () => {
|
||||
const eps = [
|
||||
ep({ schema_version: 4, review: { node_quality: 'correct' } }),
|
||||
ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:01:00Z', ended_at: '2026-05-19T10:02:00Z' }, review: { node_quality: 'correct' } }),
|
||||
ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, review: { node_quality: 'wrong_node' } }),
|
||||
];
|
||||
const res = analyze(eps);
|
||||
expect(res.reviewQuality.correct).toBe(2);
|
||||
expect(res.reviewQuality.wrong_node).toBe(1);
|
||||
expect(res.reviewerCoverage.reviewed).toBe(3);
|
||||
});
|
||||
|
||||
it('counts review pending for v4 episodes without a review block', () => {
|
||||
const eps = [ep({ schema_version: 4 })];
|
||||
expect(analyze(eps).reviewerCoverage.pending).toBe(1);
|
||||
});
|
||||
|
||||
it('counts reviewer_error escalations under reviewerCoverage.errored', () => {
|
||||
const eps = [ep({ schema_version: 4, review: { reviewer_error: 'malformed episode' } })];
|
||||
expect(analyze(eps).reviewerCoverage.errored).toBe(1);
|
||||
});
|
||||
|
||||
it('aggregates degradedCount on degraded_mode=true', () => {
|
||||
const eps = [
|
||||
ep({ schema_version: 4, degraded_mode: true }),
|
||||
ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:01:00Z', ended_at: '2026-05-19T10:02:00Z' }, degraded_mode: false }),
|
||||
];
|
||||
expect(analyze(eps).degradedCount).toBe(1);
|
||||
});
|
||||
|
||||
it('sums task_cost tokens into costTotals', () => {
|
||||
const eps = [
|
||||
ep({ schema_version: 4, task_cost: { classifier_input_tokens: 100, classifier_output_tokens: 30 } }),
|
||||
ep({ schema_version: 4, timestamps: { started_at: '2026-05-19T10:01:00Z', ended_at: '2026-05-19T10:02:00Z' }, task_cost: { classifier_input_tokens: 200, reviewer_input_tokens: 500 } }),
|
||||
];
|
||||
const ct = analyze(eps).costTotals;
|
||||
expect(ct.classifier_input_tokens).toBe(300);
|
||||
expect(ct.classifier_output_tokens).toBe(30);
|
||||
expect(ct.reviewer_input_tokens).toBe(500);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -192,7 +192,7 @@ export function buildExecutionTrace({ recommended_chain = [], invoked = [] } = {
|
||||
*/
|
||||
export function buildEpisode({ state = null, transcriptText = null, ctx = {} } = {}) {
|
||||
const base = buildEpisodeFromContext(ctx, transcriptText);
|
||||
base.schema_minor = 2; // Task 17 bump (self_assessment surface added)
|
||||
base.schema_minor = 3; // Task 20 bump (cost totals + reviewer distribution surface)
|
||||
if (state?.inheritance) {
|
||||
base.inheritance = { ...state.inheritance };
|
||||
}
|
||||
|
||||
@@ -206,10 +206,10 @@ describe('buildExecutionTrace + buildEpisode — Phase 3 Task 16 (spec §5)', ()
|
||||
expect(ep.inheritance).toBeUndefined();
|
||||
});
|
||||
|
||||
it('buildEpisode marks schema_minor=2 (Task 17 bump)', () => {
|
||||
it('buildEpisode marks schema_minor=3 (Task 20 bump)', () => {
|
||||
const ep = buildEpisode({ state: {}, ctx: { session_id: 'sess-x' } });
|
||||
expect(ep.schema_version).toBe(4);
|
||||
expect(ep.schema_minor).toBe(2);
|
||||
expect(ep.schema_minor).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user