feat(observer): brain-retro analyzer — outcome inference + factor matrix
Pure deterministic Layer-4 aggregation module (spec §6) for the /brain-retro skill. Exports: dedupeEpisodes, inferOutcome, groupEpisodesToTasks, findCausalChains, buildFactorMatrix, analyze. Read-only — never writes JSONL. 11/11 tests green. CLI smoke: 10 real episodes → valid JSON with all 5 keys. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Дмитрий
@@ -1,6 +1,6 @@
|
||||
# Brain Status (auto-generated)
|
||||
|
||||
Last updated: 2026-05-19T07:43:15.931Z
|
||||
Last updated: 2026-05-19T07:44:20.205Z
|
||||
|
||||
| Контролёр | Состояние | Детали |
|
||||
|---|---|---|
|
||||
@@ -8,7 +8,7 @@ Last updated: 2026-05-19T07:43:15.931Z
|
||||
| C2 Cross-ref consistency | ✅ | [cross-ref-checker] OK — 0 drift in 4 files |
|
||||
| C3 Observer-of-observer | ✅ | [observer-of-observer] OK — last read 0 week(s) ago |
|
||||
| C4 Сигнальный статус | ✅ | This file (self-reference) |
|
||||
| C5 Observer-coverage | ✅ | 10 episode(s), 949 recent commit(s) · Stop-hook + post-commit OK |
|
||||
| C5 Observer-coverage | ✅ | 10 episode(s), 950 recent commit(s) · Stop-hook + post-commit OK |
|
||||
|
||||
## Метрики (информационные, не алерты)
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Brain-retro analyzer (brain governance, observer factor-analysis spec §6).
|
||||
* Pure, deterministic Layer-4 aggregation over observer episodes for the
|
||||
* /brain-retro skill. Read-only — never writes JSONL. No LLM.
|
||||
*
|
||||
* Security Guidance #40: pure parsing — no exec/execSync.
|
||||
*/
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
|
||||
const SIZE_SMALL = 20;
|
||||
const SIZE_LARGE = 60;
|
||||
|
||||
/**
|
||||
* Deduplicate the routing-gate double-write: a turn that was blocked then
|
||||
* re-stopped yields two episodes with the same task_id + started_at. Keep the
|
||||
* last (most complete). observer_error markers are all kept.
|
||||
*/
|
||||
export function dedupeEpisodes(episodes) {
|
||||
const errors = episodes.filter((e) => e && e.observer_error);
|
||||
const normal = episodes.filter((e) => e && !e.observer_error);
|
||||
const byKey = new Map();
|
||||
for (const e of normal) {
|
||||
byKey.set(`${e.task_id}|${(e.timestamps || {}).started_at}`, e);
|
||||
}
|
||||
return [...byKey.values(), ...errors];
|
||||
}
|
||||
|
||||
/** Infer the true outcome of an episode from the next episode's opening prompt. */
|
||||
export function inferOutcome(episode, nextEpisode) {
|
||||
if (episode && Array.isArray(episode.events) && episode.events.some((e) => e.kind === 'interrupt')) {
|
||||
return 'partial';
|
||||
}
|
||||
if (!nextEpisode) return 'unknown';
|
||||
if (nextEpisode.prompt_signal === 'correction') return 'rework';
|
||||
if (nextEpisode.prompt_signal === 'approval' || nextEpisode.prompt_signal === 'new_task') return 'success';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
function bySessionSorted(episodes) {
|
||||
const map = new Map();
|
||||
for (const e of episodes) {
|
||||
if (e.observer_error) continue;
|
||||
const sid = e.task_id || 'unknown';
|
||||
if (!map.has(sid)) map.set(sid, []);
|
||||
map.get(sid).push(e);
|
||||
}
|
||||
for (const eps of map.values()) {
|
||||
eps.sort((a, b) =>
|
||||
String((a.timestamps || {}).started_at).localeCompare(String((b.timestamps || {}).started_at))
|
||||
);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/** Group episodes into tasks: a new task starts after a success or on a new_task prompt. */
|
||||
export function groupEpisodesToTasks(episodes) {
|
||||
const tasks = [];
|
||||
for (const [sid, eps] of bySessionSorted(episodes)) {
|
||||
let current = null;
|
||||
eps.forEach((episode, i) => {
|
||||
const prev = eps[i - 1];
|
||||
const prevOutcome = prev ? inferOutcome(prev, episode) : null;
|
||||
const isNewTask = i === 0 || prevOutcome === 'success' || episode.prompt_signal === 'new_task';
|
||||
if (isNewTask) {
|
||||
current = { task_ref: `${sid}#${tasks.length + 1}`, episodes: [] };
|
||||
tasks.push(current);
|
||||
}
|
||||
current.episodes.push(episode);
|
||||
});
|
||||
}
|
||||
return tasks;
|
||||
}
|
||||
|
||||
/** Causal-chain candidates: an errored episode → a later episode sharing a file. */
|
||||
export function findCausalChains(episodes) {
|
||||
const sorted = episodes
|
||||
.filter((e) => !e.observer_error)
|
||||
.slice()
|
||||
.sort((a, b) =>
|
||||
String((a.timestamps || {}).started_at).localeCompare(String((b.timestamps || {}).started_at))
|
||||
);
|
||||
const chains = [];
|
||||
for (let i = 0; i < sorted.length - 1; i++) {
|
||||
const a = sorted[i];
|
||||
const hasError = Array.isArray(a.events) && a.events.some((e) => e.kind === 'error');
|
||||
if (!hasError) continue;
|
||||
const filesA = new Set(((a.task_size || {}).files) || []);
|
||||
if (filesA.size === 0) continue;
|
||||
for (let j = i + 1; j < sorted.length; j++) {
|
||||
const b = sorted[j];
|
||||
const shared = (((b.task_size || {}).files) || []).filter((f) => filesA.has(f));
|
||||
if (shared.length > 0) {
|
||||
chains.push({
|
||||
from: `${a.task_id}|${(a.timestamps || {}).started_at}`,
|
||||
to: `${b.task_id}|${(b.timestamps || {}).started_at}`,
|
||||
sharedFiles: shared,
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return chains;
|
||||
}
|
||||
|
||||
function sizeBucket(toolCalls) {
|
||||
const n = Number(toolCalls) || 0;
|
||||
return n < SIZE_SMALL ? 'small' : n <= SIZE_LARGE ? 'medium' : 'large';
|
||||
}
|
||||
|
||||
const FACTOR_FNS = {
|
||||
decision_provenance: (e) => (e.decision_provenance || {}).kind || 'unknown',
|
||||
economy_level: (e) => String((e.environment || {}).economy_level ?? 'null'),
|
||||
model: (e) => (e.environment || {}).model || 'null',
|
||||
post_compaction: (e) => String((e.environment || {}).post_compaction ?? false),
|
||||
task_size: (e) => sizeBucket((e.task_size || {}).tool_calls),
|
||||
node_chosen: (e) => (e.primary_rationale || {}).node_chosen || 'direct',
|
||||
task_classification: (e) => (e.primary_rationale || {}).task_classification || 'other',
|
||||
};
|
||||
|
||||
/** Factor matrix: rows = factor values, columns = outcome distribution (spec §6). */
|
||||
export function buildFactorMatrix(episodesWithOutcome) {
|
||||
const matrix = {};
|
||||
for (const [fname, fn] of Object.entries(FACTOR_FNS)) {
|
||||
matrix[fname] = {};
|
||||
for (const e of episodesWithOutcome) {
|
||||
const val = fn(e);
|
||||
const outcome = e._inferredOutcome || 'unknown';
|
||||
matrix[fname][val] = matrix[fname][val] || {};
|
||||
matrix[fname][val][outcome] = (matrix[fname][val][outcome] || 0) + 1;
|
||||
}
|
||||
}
|
||||
return matrix;
|
||||
}
|
||||
|
||||
/** Full deterministic aggregation: dedup → infer outcomes → group → chains → matrix. */
|
||||
export function analyze(episodes) {
|
||||
const deduped = dedupeEpisodes(episodes);
|
||||
const normal = deduped.filter((e) => !e.observer_error);
|
||||
for (const eps of bySessionSorted(normal).values()) {
|
||||
eps.forEach((episode, i) => {
|
||||
episode._inferredOutcome = inferOutcome(episode, eps[i + 1]);
|
||||
});
|
||||
}
|
||||
return {
|
||||
episodeCount: normal.length,
|
||||
observerErrorCount: deduped.length - normal.length,
|
||||
tasks: groupEpisodesToTasks(normal),
|
||||
causalChains: findCausalChains(normal),
|
||||
factorMatrix: buildFactorMatrix(normal),
|
||||
};
|
||||
}
|
||||
|
||||
function loadEpisodes(files) {
|
||||
const eps = [];
|
||||
for (const f of files) {
|
||||
if (!existsSync(f)) continue;
|
||||
for (const line of readFileSync(f, 'utf-8').split('\n')) {
|
||||
const t = line.trim();
|
||||
if (!t) continue;
|
||||
try {
|
||||
eps.push(JSON.parse(t));
|
||||
} catch {
|
||||
// skip broken line
|
||||
}
|
||||
}
|
||||
}
|
||||
return eps;
|
||||
}
|
||||
|
||||
if (process.argv[1] && process.argv[1].replace(/\\/g, '/').endsWith('/brain-retro-analyzer.mjs')) {
|
||||
const result = analyze(loadEpisodes(process.argv.slice(2)));
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
dedupeEpisodes,
|
||||
inferOutcome,
|
||||
groupEpisodesToTasks,
|
||||
findCausalChains,
|
||||
buildFactorMatrix,
|
||||
analyze,
|
||||
} from './brain-retro-analyzer.mjs';
|
||||
|
||||
// Minimal v2 episode for tests.
|
||||
const ep = (overrides = {}) => ({
|
||||
schema_version: 2,
|
||||
task_id: 's1',
|
||||
task_ref: 's1',
|
||||
timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:05:00Z' },
|
||||
path_type: 'regulated',
|
||||
outcome: 'unknown',
|
||||
prompt_signal: 'neutral',
|
||||
decision_provenance: { kind: 'autonomous', claude_would_have_chosen: null },
|
||||
environment: { economy_level: 0, model: 'claude-opus-4-7', post_compaction: false, session_turn: 1, parallel_session: false },
|
||||
task_size: { tool_calls: 5, files_touched: 1, files: ['/a.js'] },
|
||||
primary_rationale: { step: 1, node_chosen: 'direct', triggers_matched: [], candidates_considered: [], boundaries_applied: [], hard_floor: { invoked: false, rules: [] }, task_classification: 'feature' },
|
||||
events: [],
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('dedupeEpisodes', () => {
|
||||
it('keeps the last of two episodes with the same task_id + started_at', () => {
|
||||
const a = ep({ outcome: 'unknown' });
|
||||
const b = ep({ outcome: 'partial' }); // same task_id + started_at — routing-gate double-write
|
||||
const out = dedupeEpisodes([a, b]);
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0].outcome).toBe('partial');
|
||||
});
|
||||
|
||||
it('keeps all observer_error markers', () => {
|
||||
const out = dedupeEpisodes([ep(), { observer_error: true, task_id: 'e' }, { observer_error: true, task_id: 'e2' }]);
|
||||
expect(out.filter((e) => e.observer_error)).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('inferOutcome', () => {
|
||||
it('infers rework when the next episode opens with a correction', () => {
|
||||
expect(inferOutcome(ep(), ep({ prompt_signal: 'correction' }))).toBe('rework');
|
||||
});
|
||||
it('infers success when the next episode opens with approval', () => {
|
||||
expect(inferOutcome(ep(), ep({ prompt_signal: 'approval' }))).toBe('success');
|
||||
});
|
||||
it('infers partial when the episode has an interrupt event', () => {
|
||||
expect(inferOutcome(ep({ events: [{ kind: 'interrupt' }] }), ep())).toBe('partial');
|
||||
});
|
||||
it('infers unknown when there is no next episode', () => {
|
||||
expect(inferOutcome(ep(), null)).toBe('unknown');
|
||||
});
|
||||
});
|
||||
|
||||
describe('groupEpisodesToTasks', () => {
|
||||
it('starts a new task after a success and on a new_task prompt', () => {
|
||||
const eps = [
|
||||
ep({ timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:01:00Z' }, prompt_signal: 'new_task' }),
|
||||
ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, prompt_signal: 'approval' }),
|
||||
ep({ timestamps: { started_at: '2026-05-19T10:04:00Z', ended_at: '2026-05-19T10:05:00Z' }, prompt_signal: 'new_task' }),
|
||||
];
|
||||
const tasks = groupEpisodesToTasks(eps);
|
||||
expect(tasks.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findCausalChains', () => {
|
||||
it('links an errored episode to a later episode that shares a file', () => {
|
||||
const a = ep({ timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:01:00Z' }, events: [{ kind: 'error', message: 'x' }], task_size: { tool_calls: 1, files_touched: 1, files: ['/shared.js'] } });
|
||||
const b = ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, task_size: { tool_calls: 1, files_touched: 1, files: ['/shared.js'] } });
|
||||
const chains = findCausalChains([a, b]);
|
||||
expect(chains).toHaveLength(1);
|
||||
expect(chains[0].sharedFiles).toEqual(['/shared.js']);
|
||||
});
|
||||
|
||||
it('returns no chain when no files are shared', () => {
|
||||
const a = ep({ events: [{ kind: 'error', message: 'x' }], task_size: { tool_calls: 1, files_touched: 1, files: ['/a.js'] } });
|
||||
const b = ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, task_size: { tool_calls: 1, files_touched: 1, files: ['/b.js'] } });
|
||||
expect(findCausalChains([a, b])).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildFactorMatrix', () => {
|
||||
it('tabulates outcome distribution per factor value', () => {
|
||||
const eps = [
|
||||
{ ...ep(), _inferredOutcome: 'rework', decision_provenance: { kind: 'user_directed_method' } },
|
||||
{ ...ep(), _inferredOutcome: 'success', decision_provenance: { kind: 'autonomous' } },
|
||||
];
|
||||
const m = buildFactorMatrix(eps);
|
||||
expect(m.decision_provenance.user_directed_method.rework).toBe(1);
|
||||
expect(m.decision_provenance.autonomous.success).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('analyze', () => {
|
||||
it('returns episodeCount, tasks, causalChains and factorMatrix', () => {
|
||||
const result = analyze([ep(), ep({ timestamps: { started_at: '2026-05-19T11:00:00Z', ended_at: '2026-05-19T11:01:00Z' }, prompt_signal: 'correction' })]);
|
||||
expect(result.episodeCount).toBe(2);
|
||||
expect(result.factorMatrix).toBeDefined();
|
||||
expect(Array.isArray(result.tasks)).toBe(true);
|
||||
expect(Array.isArray(result.causalChains)).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user