diff --git a/docs/observer/STATUS.md b/docs/observer/STATUS.md index 615a7b3e..8cd02bcb 100644 --- a/docs/observer/STATUS.md +++ b/docs/observer/STATUS.md @@ -1,6 +1,6 @@ # Brain Status (auto-generated) -Last updated: 2026-05-19T07:43:15.931Z +Last updated: 2026-05-19T07:44:20.205Z | Контролёр | Состояние | Детали | |---|---|---| @@ -8,7 +8,7 @@ Last updated: 2026-05-19T07:43:15.931Z | C2 Cross-ref consistency | ✅ | [cross-ref-checker] OK — 0 drift in 4 files | | C3 Observer-of-observer | ✅ | [observer-of-observer] OK — last read 0 week(s) ago | | C4 Сигнальный статус | ✅ | This file (self-reference) | -| C5 Observer-coverage | ✅ | 10 episode(s), 949 recent commit(s) · Stop-hook + post-commit OK | +| C5 Observer-coverage | ✅ | 10 episode(s), 950 recent commit(s) · Stop-hook + post-commit OK | ## Метрики (информационные, не алерты) diff --git a/tools/brain-retro-analyzer.mjs b/tools/brain-retro-analyzer.mjs new file mode 100644 index 00000000..762b77a3 --- /dev/null +++ b/tools/brain-retro-analyzer.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node +/** + * Brain-retro analyzer (brain governance, observer factor-analysis spec §6). + * Pure, deterministic Layer-4 aggregation over observer episodes for the + * /brain-retro skill. Read-only — never writes JSONL. No LLM. + * + * Security Guidance #40: pure parsing — no exec/execSync. + */ +import { readFileSync, existsSync } from 'fs'; + +const SIZE_SMALL = 20; +const SIZE_LARGE = 60; + +/** + * Deduplicate the routing-gate double-write: a turn that was blocked then + * re-stopped yields two episodes with the same task_id + started_at. Keep the + * last (most complete). observer_error markers are all kept. + */ +export function dedupeEpisodes(episodes) { + const errors = episodes.filter((e) => e && e.observer_error); + const normal = episodes.filter((e) => e && !e.observer_error); + const byKey = new Map(); + for (const e of normal) { + byKey.set(`${e.task_id}|${(e.timestamps || {}).started_at}`, e); + } + return [...byKey.values(), ...errors]; +} + +/** Infer the true outcome of an episode from the next episode's opening prompt. */ +export function inferOutcome(episode, nextEpisode) { + if (episode && Array.isArray(episode.events) && episode.events.some((e) => e.kind === 'interrupt')) { + return 'partial'; + } + if (!nextEpisode) return 'unknown'; + if (nextEpisode.prompt_signal === 'correction') return 'rework'; + if (nextEpisode.prompt_signal === 'approval' || nextEpisode.prompt_signal === 'new_task') return 'success'; + return 'unknown'; +} + +function bySessionSorted(episodes) { + const map = new Map(); + for (const e of episodes) { + if (e.observer_error) continue; + const sid = e.task_id || 'unknown'; + if (!map.has(sid)) map.set(sid, []); + map.get(sid).push(e); + } + for (const eps of map.values()) { + eps.sort((a, b) => + String((a.timestamps || {}).started_at).localeCompare(String((b.timestamps || {}).started_at)) + ); + } + return map; +} + +/** Group episodes into tasks: a new task starts after a success or on a new_task prompt. */ +export function groupEpisodesToTasks(episodes) { + const tasks = []; + for (const [sid, eps] of bySessionSorted(episodes)) { + let current = null; + eps.forEach((episode, i) => { + const prev = eps[i - 1]; + const prevOutcome = prev ? inferOutcome(prev, episode) : null; + const isNewTask = i === 0 || prevOutcome === 'success' || episode.prompt_signal === 'new_task'; + if (isNewTask) { + current = { task_ref: `${sid}#${tasks.length + 1}`, episodes: [] }; + tasks.push(current); + } + current.episodes.push(episode); + }); + } + return tasks; +} + +/** Causal-chain candidates: an errored episode → a later episode sharing a file. */ +export function findCausalChains(episodes) { + const sorted = episodes + .filter((e) => !e.observer_error) + .slice() + .sort((a, b) => + String((a.timestamps || {}).started_at).localeCompare(String((b.timestamps || {}).started_at)) + ); + const chains = []; + for (let i = 0; i < sorted.length - 1; i++) { + const a = sorted[i]; + const hasError = Array.isArray(a.events) && a.events.some((e) => e.kind === 'error'); + if (!hasError) continue; + const filesA = new Set(((a.task_size || {}).files) || []); + if (filesA.size === 0) continue; + for (let j = i + 1; j < sorted.length; j++) { + const b = sorted[j]; + const shared = (((b.task_size || {}).files) || []).filter((f) => filesA.has(f)); + if (shared.length > 0) { + chains.push({ + from: `${a.task_id}|${(a.timestamps || {}).started_at}`, + to: `${b.task_id}|${(b.timestamps || {}).started_at}`, + sharedFiles: shared, + }); + break; + } + } + } + return chains; +} + +function sizeBucket(toolCalls) { + const n = Number(toolCalls) || 0; + return n < SIZE_SMALL ? 'small' : n <= SIZE_LARGE ? 'medium' : 'large'; +} + +const FACTOR_FNS = { + decision_provenance: (e) => (e.decision_provenance || {}).kind || 'unknown', + economy_level: (e) => String((e.environment || {}).economy_level ?? 'null'), + model: (e) => (e.environment || {}).model || 'null', + post_compaction: (e) => String((e.environment || {}).post_compaction ?? false), + task_size: (e) => sizeBucket((e.task_size || {}).tool_calls), + node_chosen: (e) => (e.primary_rationale || {}).node_chosen || 'direct', + task_classification: (e) => (e.primary_rationale || {}).task_classification || 'other', +}; + +/** Factor matrix: rows = factor values, columns = outcome distribution (spec §6). */ +export function buildFactorMatrix(episodesWithOutcome) { + const matrix = {}; + for (const [fname, fn] of Object.entries(FACTOR_FNS)) { + matrix[fname] = {}; + for (const e of episodesWithOutcome) { + const val = fn(e); + const outcome = e._inferredOutcome || 'unknown'; + matrix[fname][val] = matrix[fname][val] || {}; + matrix[fname][val][outcome] = (matrix[fname][val][outcome] || 0) + 1; + } + } + return matrix; +} + +/** Full deterministic aggregation: dedup → infer outcomes → group → chains → matrix. */ +export function analyze(episodes) { + const deduped = dedupeEpisodes(episodes); + const normal = deduped.filter((e) => !e.observer_error); + for (const eps of bySessionSorted(normal).values()) { + eps.forEach((episode, i) => { + episode._inferredOutcome = inferOutcome(episode, eps[i + 1]); + }); + } + return { + episodeCount: normal.length, + observerErrorCount: deduped.length - normal.length, + tasks: groupEpisodesToTasks(normal), + causalChains: findCausalChains(normal), + factorMatrix: buildFactorMatrix(normal), + }; +} + +function loadEpisodes(files) { + const eps = []; + for (const f of files) { + if (!existsSync(f)) continue; + for (const line of readFileSync(f, 'utf-8').split('\n')) { + const t = line.trim(); + if (!t) continue; + try { + eps.push(JSON.parse(t)); + } catch { + // skip broken line + } + } + } + return eps; +} + +if (process.argv[1] && process.argv[1].replace(/\\/g, '/').endsWith('/brain-retro-analyzer.mjs')) { + const result = analyze(loadEpisodes(process.argv.slice(2))); + console.log(JSON.stringify(result, null, 2)); + process.exit(0); +} diff --git a/tools/brain-retro-analyzer.test.mjs b/tools/brain-retro-analyzer.test.mjs new file mode 100644 index 00000000..de69e335 --- /dev/null +++ b/tools/brain-retro-analyzer.test.mjs @@ -0,0 +1,106 @@ +import { describe, it, expect } from 'vitest'; +import { + dedupeEpisodes, + inferOutcome, + groupEpisodesToTasks, + findCausalChains, + buildFactorMatrix, + analyze, +} from './brain-retro-analyzer.mjs'; + +// Minimal v2 episode for tests. +const ep = (overrides = {}) => ({ + schema_version: 2, + task_id: 's1', + task_ref: 's1', + timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:05:00Z' }, + path_type: 'regulated', + outcome: 'unknown', + prompt_signal: 'neutral', + decision_provenance: { kind: 'autonomous', claude_would_have_chosen: null }, + environment: { economy_level: 0, model: 'claude-opus-4-7', post_compaction: false, session_turn: 1, parallel_session: false }, + task_size: { tool_calls: 5, files_touched: 1, files: ['/a.js'] }, + primary_rationale: { step: 1, node_chosen: 'direct', triggers_matched: [], candidates_considered: [], boundaries_applied: [], hard_floor: { invoked: false, rules: [] }, task_classification: 'feature' }, + events: [], + ...overrides, +}); + +describe('dedupeEpisodes', () => { + it('keeps the last of two episodes with the same task_id + started_at', () => { + const a = ep({ outcome: 'unknown' }); + const b = ep({ outcome: 'partial' }); // same task_id + started_at — routing-gate double-write + const out = dedupeEpisodes([a, b]); + expect(out).toHaveLength(1); + expect(out[0].outcome).toBe('partial'); + }); + + it('keeps all observer_error markers', () => { + const out = dedupeEpisodes([ep(), { observer_error: true, task_id: 'e' }, { observer_error: true, task_id: 'e2' }]); + expect(out.filter((e) => e.observer_error)).toHaveLength(2); + }); +}); + +describe('inferOutcome', () => { + it('infers rework when the next episode opens with a correction', () => { + expect(inferOutcome(ep(), ep({ prompt_signal: 'correction' }))).toBe('rework'); + }); + it('infers success when the next episode opens with approval', () => { + expect(inferOutcome(ep(), ep({ prompt_signal: 'approval' }))).toBe('success'); + }); + it('infers partial when the episode has an interrupt event', () => { + expect(inferOutcome(ep({ events: [{ kind: 'interrupt' }] }), ep())).toBe('partial'); + }); + it('infers unknown when there is no next episode', () => { + expect(inferOutcome(ep(), null)).toBe('unknown'); + }); +}); + +describe('groupEpisodesToTasks', () => { + it('starts a new task after a success and on a new_task prompt', () => { + const eps = [ + ep({ timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:01:00Z' }, prompt_signal: 'new_task' }), + ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, prompt_signal: 'approval' }), + ep({ timestamps: { started_at: '2026-05-19T10:04:00Z', ended_at: '2026-05-19T10:05:00Z' }, prompt_signal: 'new_task' }), + ]; + const tasks = groupEpisodesToTasks(eps); + expect(tasks.length).toBeGreaterThanOrEqual(2); + }); +}); + +describe('findCausalChains', () => { + it('links an errored episode to a later episode that shares a file', () => { + const a = ep({ timestamps: { started_at: '2026-05-19T10:00:00Z', ended_at: '2026-05-19T10:01:00Z' }, events: [{ kind: 'error', message: 'x' }], task_size: { tool_calls: 1, files_touched: 1, files: ['/shared.js'] } }); + const b = ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, task_size: { tool_calls: 1, files_touched: 1, files: ['/shared.js'] } }); + const chains = findCausalChains([a, b]); + expect(chains).toHaveLength(1); + expect(chains[0].sharedFiles).toEqual(['/shared.js']); + }); + + it('returns no chain when no files are shared', () => { + const a = ep({ events: [{ kind: 'error', message: 'x' }], task_size: { tool_calls: 1, files_touched: 1, files: ['/a.js'] } }); + const b = ep({ timestamps: { started_at: '2026-05-19T10:02:00Z', ended_at: '2026-05-19T10:03:00Z' }, task_size: { tool_calls: 1, files_touched: 1, files: ['/b.js'] } }); + expect(findCausalChains([a, b])).toHaveLength(0); + }); +}); + +describe('buildFactorMatrix', () => { + it('tabulates outcome distribution per factor value', () => { + const eps = [ + { ...ep(), _inferredOutcome: 'rework', decision_provenance: { kind: 'user_directed_method' } }, + { ...ep(), _inferredOutcome: 'success', decision_provenance: { kind: 'autonomous' } }, + ]; + const m = buildFactorMatrix(eps); + expect(m.decision_provenance.user_directed_method.rework).toBe(1); + expect(m.decision_provenance.autonomous.success).toBe(1); + }); +}); + +describe('analyze', () => { + it('returns episodeCount, tasks, causalChains and factorMatrix', () => { + const result = analyze([ep(), ep({ timestamps: { started_at: '2026-05-19T11:00:00Z', ended_at: '2026-05-19T11:01:00Z' }, prompt_signal: 'correction' })]); + expect(result.episodeCount).toBe(2); + expect(result.factorMatrix).toBeDefined(); + expect(Array.isArray(result.tasks)).toBe(true); + expect(Array.isArray(result.causalChains)).toBe(true); + }); +});