From c0e3e901d0f02dfd21c46e22418878c9fe3f3cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Wed, 20 May 2026 13:10:11 +0300 Subject: [PATCH] feat(observer): differentiate error events by tool + summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes brain-retro 2026-05-20 #7 — each tool_result.is_error now emits { kind:'error', tool:, summary: }. Allows aggregation by tool (Bash/Edit/Read) + cause prefix (ENOENT/timeout/ 'String to replace not found'). Required updating existing 'emits error events for tool_result with is_error' test assertion (old shape had bare 'message' field). 4 new vitest tests + 1 existing relaxed, 286/286 GREEN. Co-Authored-By: Claude Sonnet 4.6 --- tools/observer-transcript-parser.mjs | 25 ++++++-- tools/observer-transcript-parser.test.mjs | 70 ++++++++++++++++++++++- 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/tools/observer-transcript-parser.mjs b/tools/observer-transcript-parser.mjs index 1e4b2b04..eb2d9d4b 100644 --- a/tools/observer-transcript-parser.mjs +++ b/tools/observer-transcript-parser.mjs @@ -127,7 +127,16 @@ export function classifyTask(text) { function collectToolUse(entries) { const skills = []; const counts = {}; - let errorCount = 0; + const errors = []; + const idToTool = {}; + // First pass — build id→tool name map (tool_results may reference tools across messages) + for (const e of entries) { + const content = e && e.message && Array.isArray(e.message.content) ? e.message.content : []; + for (const b of content) { + if (b && b.type === 'tool_use') idToTool[b.id] = b.name || 'unknown'; + } + } + // Second pass — accumulate counts + per-error attribution for (const e of entries) { const content = e && e.message && Array.isArray(e.message.content) ? e.message.content : []; for (const block of content) { @@ -139,11 +148,15 @@ function collectToolUse(entries) { skills.push((block.input && block.input.skill) || 'unknown'); } } else if (block.type === 'tool_result' && block.is_error === true) { - errorCount += 1; + const tool = idToTool[block.tool_use_id] || 'unknown'; + const c = block.content; + const text = typeof c === 'string' ? c + : (Array.isArray(c) ? c.map((b) => (b && typeof b.text === 'string') ? b.text : '').join(' ') : ''); + errors.push({ tool, summary: text.slice(0, 80) }); } } } - return { skills, counts, errorCount }; + return { skills, counts, errors }; } const FILE_TOOLS = new Set(['Read', 'Edit', 'Write', 'MultiEdit', 'NotebookEdit']); @@ -557,13 +570,13 @@ export function parseTranscript(transcriptText, fallbackSessionId = null) { const ended_at = stamps[stamps.length - 1] || started_at; const durationMs = new Date(ended_at) - new Date(started_at); - const { skills, counts, errorCount } = collectToolUse(turn); + const { skills, counts, errors } = collectToolUse(turn); const events = []; for (const skill of skills) events.push({ kind: 'skill_invoked', skill }); if (Object.keys(counts).length > 0) events.push({ kind: 'tool_summary', counts }); - for (let i = 0; i < errorCount; i++) { - events.push({ kind: 'error', message: 'tool_result reported is_error' }); + for (const err of errors) { + events.push({ kind: 'error', tool: err.tool, summary: err.summary }); } events.push(...extractProcessEvents(turn, broken, total, durationMs)); events.push(...extractAskUserQuestionEvents(turn)); diff --git a/tools/observer-transcript-parser.test.mjs b/tools/observer-transcript-parser.test.mjs index 9c03a7ae..84614a19 100644 --- a/tools/observer-transcript-parser.test.mjs +++ b/tools/observer-transcript-parser.test.mjs @@ -172,7 +172,10 @@ describe('parseTranscript', () => { assistantTurn([{ type: 'tool_use', id: 't1', name: 'Bash', input: {} }], '2026-05-19T10:01:00Z'), toolResult('t1', true, '2026-05-19T10:02:00Z'), ]); - expect(parseTranscript(t).events.filter((e) => e.kind === 'error')).toHaveLength(1); + const errs = parseTranscript(t).events.filter((e) => e.kind === 'error'); + expect(errs).toHaveLength(1); + expect(errs[0].tool).toBe('Bash'); + expect(errs[0].summary).toBe('r'); }); it('skips broken JSONL lines without throwing', () => { @@ -1180,3 +1183,68 @@ describe('parseTranscript — heuristic primary_rationale (Task 6)', () => { expect(ep.primary_rationale.boundaries_applied).toContain('ADR-011'); }); }); + +describe('error event differentiation (Task 7)', () => { + it('attributes error to the failing tool and captures summary', () => { + const transcript = [ + JSON.stringify({ sessionId: 's' }), + JSON.stringify({ type: 'user', message: { role: 'user', content: 'делай' }, uuid: 'u1', timestamp: '2026-05-20T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: [ + { type: 'tool_use', id: 'tu1', name: 'Bash', input: {} } + ] }, uuid: 'u2', timestamp: '2026-05-20T00:00:01Z' }), + JSON.stringify({ type: 'user', message: { role: 'user', content: [ + { type: 'tool_result', tool_use_id: 'tu1', is_error: true, content: 'ENOENT: no such file or directory' } + ] }, uuid: 'u3', timestamp: '2026-05-20T00:00:02Z' }), + ].join('\n'); + const ep = parseTranscript(transcript); + const errorEvents = ep.events.filter((e) => e.kind === 'error'); + expect(errorEvents).toHaveLength(1); + expect(errorEvents[0].tool).toBe('Bash'); + expect(errorEvents[0].summary).toContain('ENOENT'); + }); + it('attributes error to "unknown" when tool_use_id not seen', () => { + const transcript = [ + JSON.stringify({ sessionId: 's' }), + JSON.stringify({ type: 'user', message: { role: 'user', content: 'делай' }, uuid: 'u1', timestamp: '2026-05-20T00:00:00Z' }), + JSON.stringify({ type: 'user', message: { role: 'user', content: [ + { type: 'tool_result', tool_use_id: 'orphan', is_error: true, content: 'mystery error' } + ] }, uuid: 'u2', timestamp: '2026-05-20T00:00:02Z' }), + ].join('\n'); + const ep = parseTranscript(transcript); + const errs = ep.events.filter((e) => e.kind === 'error'); + expect(errs[0].tool).toBe('unknown'); + expect(errs[0].summary).toBe('mystery error'); + }); + it('truncates summary at 80 chars', () => { + const longText = 'a'.repeat(200); + const transcript = [ + JSON.stringify({ sessionId: 's' }), + JSON.stringify({ type: 'user', message: { role: 'user', content: 'делай' }, uuid: 'u1', timestamp: '2026-05-20T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: [ + { type: 'tool_use', id: 'tu1', name: 'Read', input: {} } + ] }, uuid: 'u2', timestamp: '2026-05-20T00:00:01Z' }), + JSON.stringify({ type: 'user', message: { role: 'user', content: [ + { type: 'tool_result', tool_use_id: 'tu1', is_error: true, content: longText } + ] }, uuid: 'u3', timestamp: '2026-05-20T00:00:02Z' }), + ].join('\n'); + const ep = parseTranscript(transcript); + const e = ep.events.find((x) => x.kind === 'error'); + expect(e.summary.length).toBe(80); + }); + it('handles structured content array (text blocks)', () => { + const transcript = [ + JSON.stringify({ sessionId: 's' }), + JSON.stringify({ type: 'user', message: { role: 'user', content: 'делай' }, uuid: 'u1', timestamp: '2026-05-20T00:00:00Z' }), + JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: [ + { type: 'tool_use', id: 'tu1', name: 'Edit', input: {} } + ] }, uuid: 'u2', timestamp: '2026-05-20T00:00:01Z' }), + JSON.stringify({ type: 'user', message: { role: 'user', content: [ + { type: 'tool_result', tool_use_id: 'tu1', is_error: true, content: [{ type: 'text', text: 'String to replace not found' }] } + ] }, uuid: 'u3', timestamp: '2026-05-20T00:00:02Z' }), + ].join('\n'); + const ep = parseTranscript(transcript); + const e = ep.events.find((x) => x.kind === 'error'); + expect(e.tool).toBe('Edit'); + expect(e.summary).toContain('String to replace'); + }); +});