Дмитрий
292 lines
14 KiB
JavaScript
292 lines
14 KiB
JavaScript
import { describe, it, expect } from 'vitest';
|
|
import {
|
|
dedupeEpisodes,
|
|
inferOutcome,
|
|
groupEpisodesToTasks,
|
|
findCausalChains,
|
|
buildFactorMatrix,
|
|
analyze,
|
|
} from './brain-retro-analyzer.mjs';
|
|
|
|
// Minimal v2 episode for tests.
|
|
const ep = (overrides = {}) => ({
|
|
schema_version: 2,
|
|
task_id: 's1',
|
|
task_ref: 's1',
|
|
timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:05:00Z' },
|
|
path_type: 'regulated',
|
|
outcome: 'unknown',
|
|
prompt_signal: 'neutral',
|
|
decision_provenance: { kind: 'autonomous', claude_would_have_chosen: null },
|
|
environment: { economy_level: 0, model: 'claude-opus-4-7', post_compaction: false, session_turn: 1, parallel_session: false },
|
|
task_size: { tool_calls: 5, files_touched: 1, files: ['/a.js'] },
|
|
primary_rationale: { step: 1, node_chosen: 'direct', triggers_matched: [], candidates_considered: [], boundaries_applied: [], hard_floor: { invoked: false, rules: [] }, task_classification: 'feature' },
|
|
events: [],
|
|
...overrides,
|
|
});
|
|
|
|
describe('dedupeEpisodes', () => {
|
|
it('keeps the last of two episodes with the same task_id + started_at', () => {
|
|
const a = ep({ outcome: 'unknown' });
|
|
const b = ep({ outcome: 'partial' }); // same task_id + started_at — routing-gate double-write
|
|
const out = dedupeEpisodes([a, b]);
|
|
expect(out).toHaveLength(1);
|
|
expect(out[0].outcome).toBe('partial');
|
|
});
|
|
|
|
it('keeps all observer_error markers', () => {
|
|
const out = dedupeEpisodes([ep(), { observer_error: true, task_id: 'e' }, { observer_error: true, task_id: 'e2' }]);
|
|
expect(out.filter((e) => e.observer_error)).toHaveLength(2);
|
|
});
|
|
});
|
|
|
|
describe('inferOutcome', () => {
|
|
it('infers rework when the next episode opens with a correction', () => {
|
|
expect(inferOutcome(ep(), ep({ prompt_signal: 'correction' }))).toBe('rework');
|
|
});
|
|
it('infers success when the next episode opens with approval', () => {
|
|
expect(inferOutcome(ep(), ep({ prompt_signal: 'approval' }))).toBe('success');
|
|
});
|
|
it('infers partial when the episode has an interrupt event', () => {
|
|
expect(inferOutcome(ep({ events: [{ kind: 'interrupt' }] }), ep())).toBe('partial');
|
|
});
|
|
it('infers unknown when there is no next episode', () => {
|
|
expect(inferOutcome(ep(), null)).toBe('unknown');
|
|
});
|
|
it('infers blocked ONLY when an unrecovered_error event is present (turn ended on error)', () => {
|
|
const blocked = ep({ events: [{ kind: 'error' }, { kind: 'error' }, { kind: 'unrecovered_error' }] });
|
|
expect(inferOutcome(blocked, ep({ prompt_signal: 'approval' }))).toBe('blocked');
|
|
});
|
|
it('does NOT infer blocked from raw error/retry count (TDD failing-test-first is not a block)', () => {
|
|
// A turn with N errors + N retries that ends on a successful tool_result —
|
|
// e.g., TDD red→green, or git command that legitimately fails then recovers —
|
|
// must NOT count as blocked. The parser emits unrecovered_error iff the LAST
|
|
// tool_result was is_error, which is absent here.
|
|
const recovered = ep({ events: [{ kind: 'error' }, { kind: 'error' }, { kind: 'retry' }] });
|
|
expect(inferOutcome(recovered, ep({ prompt_signal: 'approval' }))).toBe('success');
|
|
});
|
|
it('does not infer blocked when every error was retried', () => {
|
|
const recovered = ep({ events: [{ kind: 'error' }, { kind: 'retry' }] });
|
|
expect(inferOutcome(recovered, ep({ prompt_signal: 'approval' }))).toBe('success');
|
|
});
|
|
});
|
|
|
|
describe('groupEpisodesToTasks', () => {
|
|
it('starts a new task after a success and on a new_task prompt', () => {
|
|
const eps = [
|
|
ep({ timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:01:00Z' }, prompt_signal: 'new_task' }),
|
|
ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, prompt_signal: 'approval' }),
|
|
ep({ timestamps: { started_at: '2026-05-19T10:04:00Z', ended_at: '2026-05-19T10:05:00Z' }, prompt_signal: 'new_task' }),
|
|
];
|
|
const tasks = groupEpisodesToTasks(eps);
|
|
expect(tasks.length).toBeGreaterThanOrEqual(2);
|
|
});
|
|
});
|
|
|
|
describe('findCausalChains', () => {
|
|
it('links an errored episode to a later episode that shares a file', () => {
|
|
const a = ep({ timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:01:00Z' }, events: [{ kind: 'error', message: 'x' }], task_size: { tool_calls: 1, files_touched: 1, files: ['/shared.js'] } });
|
|
const b = ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, task_size: { tool_calls: 1, files_touched: 1, files: ['/shared.js'] } });
|
|
const chains = findCausalChains([a, b]);
|
|
expect(chains).toHaveLength(1);
|
|
expect(chains[0].sharedFiles).toEqual(['/shared.js']);
|
|
});
|
|
|
|
it('returns no chain when no files are shared', () => {
|
|
const a = ep({ events: [{ kind: 'error', message: 'x' }], task_size: { tool_calls: 1, files_touched: 1, files: ['/a.js'] } });
|
|
const b = ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, task_size: { tool_calls: 1, files_touched: 1, files: ['/b.js'] } });
|
|
expect(findCausalChains([a, b])).toHaveLength(0);
|
|
});
|
|
|
|
it('excludes hot/normative files (CLAUDE.md) from the shared-file signal', () => {
|
|
const a = ep({
|
|
events: [{ kind: 'error', message: 'x' }],
|
|
task_size: { tool_calls: 1, files_touched: 1, files: ['c:\\моя\\проекты\\портал crm\\Документация\\CLAUDE.md'] },
|
|
});
|
|
const b = ep({
|
|
timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' },
|
|
task_size: { tool_calls: 1, files_touched: 1, files: ['c:\\моя\\проекты\\портал crm\\Документация\\CLAUDE.md'] },
|
|
});
|
|
expect(findCausalChains([a, b])).toHaveLength(0);
|
|
});
|
|
|
|
it('excludes memory store .md files from the shared-file signal', () => {
|
|
const a = ep({
|
|
events: [{ kind: 'error', message: 'x' }],
|
|
task_size: { tool_calls: 1, files_touched: 1, files: ['C:\\Users\\Administrator\\.claude\\projects\\proj\\memory\\reference_github.md'] },
|
|
});
|
|
const b = ep({
|
|
timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' },
|
|
task_size: { tool_calls: 1, files_touched: 1, files: ['C:\\Users\\Administrator\\.claude\\projects\\proj\\memory\\reference_github.md'] },
|
|
});
|
|
expect(findCausalChains([a, b])).toHaveLength(0);
|
|
});
|
|
|
|
it('excludes episodes JSONL + STATUS.md + MEMORY.md from chains', () => {
|
|
const mk = (path, evts = []) =>
|
|
ep({
|
|
timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:01:00Z' },
|
|
events: evts,
|
|
task_size: { tool_calls: 1, files_touched: 1, files: [path] },
|
|
});
|
|
const later = (path) =>
|
|
ep({
|
|
timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' },
|
|
task_size: { tool_calls: 1, files_touched: 1, files: [path] },
|
|
});
|
|
const errored = [{ kind: 'error', message: 'x' }];
|
|
expect(findCausalChains([mk('/docs/observer/episodes-2026-05.jsonl', errored), later('/docs/observer/episodes-2026-05.jsonl')])).toHaveLength(0);
|
|
expect(findCausalChains([mk('/docs/observer/STATUS.md', errored), later('/docs/observer/STATUS.md')])).toHaveLength(0);
|
|
expect(findCausalChains([mk('/some/dir/MEMORY.md', errored), later('/some/dir/MEMORY.md')])).toHaveLength(0);
|
|
});
|
|
|
|
it('still links chains via genuinely-shared source files', () => {
|
|
const a = ep({
|
|
events: [{ kind: 'error', message: 'x' }],
|
|
task_size: { tool_calls: 1, files_touched: 2, files: ['c:\\path\\CLAUDE.md', '/src/app.ts'] },
|
|
});
|
|
const b = ep({
|
|
timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' },
|
|
task_size: { tool_calls: 1, files_touched: 2, files: ['c:\\path\\CLAUDE.md', '/src/app.ts'] },
|
|
});
|
|
const chains = findCausalChains([a, b]);
|
|
expect(chains).toHaveLength(1);
|
|
expect(chains[0].sharedFiles).toEqual(['/src/app.ts']);
|
|
});
|
|
});
|
|
|
|
describe('buildFactorMatrix', () => {
|
|
it('tabulates outcome distribution per factor value', () => {
|
|
const eps = [
|
|
{ ...ep(), _inferredOutcome: 'rework', decision_provenance: { kind: 'user_directed_method' } },
|
|
{ ...ep(), _inferredOutcome: 'success', decision_provenance: { kind: 'autonomous' } },
|
|
];
|
|
const m = buildFactorMatrix(eps);
|
|
expect(m.decision_provenance.user_directed_method.rework).toBe(1);
|
|
expect(m.decision_provenance.autonomous.success).toBe(1);
|
|
});
|
|
|
|
it('counts the 3rd kind user_chose_from_options on the provenance axis', () => {
|
|
const eps = [
|
|
{ ...ep(), _inferredOutcome: 'success', decision_provenance: { kind: 'autonomous' } },
|
|
{ ...ep(), _inferredOutcome: 'rework', decision_provenance: { kind: 'user_directed_method' } },
|
|
{ ...ep(), _inferredOutcome: 'success', decision_provenance: { kind: 'user_chose_from_options' } },
|
|
{ ...ep(), _inferredOutcome: 'rework', decision_provenance: { kind: 'user_chose_from_options' } },
|
|
];
|
|
const m = buildFactorMatrix(eps);
|
|
expect(m.decision_provenance).toHaveProperty('autonomous');
|
|
expect(m.decision_provenance).toHaveProperty('user_directed_method');
|
|
expect(m.decision_provenance).toHaveProperty('user_chose_from_options');
|
|
expect(m.decision_provenance.user_chose_from_options.success).toBe(1);
|
|
expect(m.decision_provenance.user_chose_from_options.rework).toBe(1);
|
|
});
|
|
|
|
it('includes session_segment_turn (bucketed, turns-since-last-compaction) and parallel_session factors', () => {
|
|
const eps = [
|
|
{ ...ep(), _inferredOutcome: 'success', environment: { session_turn: 3, parallel_session: false } },
|
|
{ ...ep(), _inferredOutcome: 'rework', environment: { session_turn: 120, parallel_session: true } },
|
|
];
|
|
const m = buildFactorMatrix(eps);
|
|
expect(m.session_segment_turn.early.success).toBe(1);
|
|
expect(m.session_segment_turn.late.rework).toBe(1);
|
|
expect(m.parallel_session.false.success).toBe(1);
|
|
expect(m.parallel_session.true.rework).toBe(1);
|
|
});
|
|
});
|
|
|
|
describe('analyze', () => {
|
|
it('returns episodeCount, tasks, causalChains and factorMatrix', () => {
|
|
const result = analyze([ep(), ep({ timestamps: { started_at: '2026-05-19T11:00:00Z', ended_at: '2026-05-19T11:01:00Z' }, prompt_signal: 'correction' })]);
|
|
expect(result.episodeCount).toBe(2);
|
|
expect(result.factorMatrix).toBeDefined();
|
|
expect(Array.isArray(result.tasks)).toBe(true);
|
|
expect(Array.isArray(result.causalChains)).toBe(true);
|
|
});
|
|
|
|
it('skips v1 episodes (no schema_version 2) from the analysis', () => {
|
|
const v1 = { task_id: 's-old', timestamps: { started_at: '2026-05-19T09:00:00Z' }, outcome: 'success' };
|
|
const result = analyze([
|
|
v1,
|
|
ep(),
|
|
ep({ timestamps: { started_at: '2026-05-19T11:00:00Z', ended_at: '2026-05-19T11:01:00Z' } }),
|
|
]);
|
|
expect(result.episodeCount).toBe(2);
|
|
expect(result.v1SkippedCount).toBe(1);
|
|
});
|
|
});
|
|
|
|
describe('buildFactorMatrix — session_segment_turn axis rename (Task 14)', () => {
|
|
it('matrix has session_segment_turn axis, NOT legacy session_turn', () => {
|
|
const result = analyze([
|
|
{ schema_version: 2, task_id: 's', task_ref: 's',
|
|
timestamps: { started_at: '2026-05-20T00:00:00Z' }, events: [],
|
|
environment: { economy_level: null, model: 'opus', post_compaction: false, session_turn: 5, parallel_session: false },
|
|
task_size: { tool_calls: 0 },
|
|
primary_rationale: { node_chosen: 'direct', task_classification: 'other' },
|
|
decision_provenance: { kind: 'autonomous' } },
|
|
]);
|
|
expect(result.factorMatrix).toHaveProperty('session_segment_turn');
|
|
expect(result.factorMatrix).not.toHaveProperty('session_turn');
|
|
});
|
|
});
|
|
|
|
describe('buildFactorMatrix — chain_ref axis (multi-chain)', () => {
|
|
it('counts a multi-chain episode in each chain and null for direct', () => {
|
|
const m = buildFactorMatrix([
|
|
{ _inferredOutcome: 'success', primary_rationale: { node_chosen: 'discovery-interview', chain_ref: ['L1', 'L2'] } },
|
|
{ _inferredOutcome: 'unknown', primary_rationale: { node_chosen: 'direct', chain_ref: null } },
|
|
]);
|
|
expect(m.chain_ref.L1).toEqual({ success: 1 });
|
|
expect(m.chain_ref.L2).toEqual({ success: 1 });
|
|
expect(m.chain_ref.null).toEqual({ unknown: 1 });
|
|
});
|
|
|
|
it('chain_ref axis present via analyze()', () => {
|
|
const result = analyze([ep({ primary_rationale: { node_chosen: 'billing-audit', chain_ref: ['L13'], task_classification: 'other' } })]);
|
|
expect(result.factorMatrix).toHaveProperty('chain_ref');
|
|
});
|
|
});
|
|
|
|
describe('inferOutcome — neutral → soft_success (Task 16)', () => {
|
|
it('returns soft_success when next prompt is neutral', () => {
|
|
const a = { events: [] };
|
|
const b = { prompt_signal: 'neutral' };
|
|
expect(inferOutcome(a, b)).toBe('soft_success');
|
|
});
|
|
it('returns unknown when no next episode', () => {
|
|
expect(inferOutcome({ events: [] }, null)).toBe('unknown');
|
|
});
|
|
it('rework still wins over neutral on correction', () => {
|
|
expect(inferOutcome({ events: [] }, { prompt_signal: 'correction' })).toBe('rework');
|
|
});
|
|
it('explicit success still wins over neutral on approval', () => {
|
|
expect(inferOutcome({ events: [] }, { prompt_signal: 'approval' })).toBe('success');
|
|
});
|
|
});
|
|
|
|
describe('analyze() — missedActivations integration', () => {
|
|
it('includes missedActivations in the result', () => {
|
|
const eps = [
|
|
{
|
|
schema_version: 2,
|
|
task_id: 't1',
|
|
timestamps: { started_at: '2026-05-21T00:00:00Z' },
|
|
primary_rationale: { node_chosen: 'direct', task_classification: 'refactor' },
|
|
events: [],
|
|
},
|
|
];
|
|
const map = { refactor: ['#11'], other: [] };
|
|
const dormancy = { '#11': false };
|
|
const result = analyze(eps, { classificationMap: map, dormancy });
|
|
expect(result.missedActivations).toBeDefined();
|
|
expect(result.missedActivations.totalMissed).toBe(1);
|
|
expect(result.missedActivations.byNode).toEqual({ '#11': 1 });
|
|
});
|
|
|
|
it('returns missedActivations.totalMissed=0 when no map/dormancy provided', () => {
|
|
const eps = [{ schema_version: 2, task_id: 't1', timestamps: { started_at: 'x' }, primary_rationale: { node_chosen: 'direct', task_classification: 'refactor' }, events: [] }];
|
|
const result = analyze(eps);
|
|
expect(result.missedActivations.totalMissed).toBe(0);
|
|
});
|
|
});
|