liderra/portal
1
0
Fork 0
Code Issues Pull Requests Actions 65 Packages Projects Releases Wiki Activity

feat(observer): differentiate error events by tool + summary

Browse Source 
Closes brain-retro 2026-05-20 #7 — each tool_result.is_error now emits
{ kind:'error', tool:<name>, summary:<first 80 chars> }. Allows
aggregation by tool (Bash/Edit/Read) + cause prefix (ENOENT/timeout/
'String to replace not found').

Required updating existing 'emits error events for tool_result with
is_error' test assertion (old shape had bare 'message' field).

4 new vitest tests + 1 existing relaxed, 286/286 GREEN.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Дмитрий
2026-05-20 13:10:11 +03:00
parent 0663479bb8
commit c0e3e901d0
2 changed files with 88 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/observer-transcript-parser.mjs
+19 -6
View File
@@ -127,7 +127,16 @@ export function classifyTask(text) {
function collectToolUse(entries) {
const skills = [];
const counts = {};
let errorCount = 0;
const errors = [];
const idToTool = {};
// First pass — build id→tool name map (tool_results may reference tools across messages)
for (const e of entries) {
const content = e && e.message && Array.isArray(e.message.content) ? e.message.content : [];
for (const b of content) {
if (b && b.type === 'tool_use') idToTool[b.id] = b.name || 'unknown';
}
}
// Second pass — accumulate counts + per-error attribution
for (const e of entries) {
const content = e && e.message && Array.isArray(e.message.content) ? e.message.content : [];
for (const block of content) {
@@ -139,11 +148,15 @@ function collectToolUse(entries) {
skills.push((block.input && block.input.skill) || 'unknown');
}
} else if (block.type === 'tool_result' && block.is_error === true) {
errorCount += 1;
const tool = idToTool[block.tool_use_id] || 'unknown';
const c = block.content;
const text = typeof c === 'string' ? c
: (Array.isArray(c) ? c.map((b) => (b && typeof b.text === 'string') ? b.text : '').join(' ') : '');
errors.push({ tool, summary: text.slice(0, 80) });
}
}
}
return { skills, counts, errorCount };
return { skills, counts, errors };
}
const FILE_TOOLS = new Set(['Read', 'Edit', 'Write', 'MultiEdit', 'NotebookEdit']);
@@ -557,13 +570,13 @@ export function parseTranscript(transcriptText, fallbackSessionId = null) {
const ended_at = stamps[stamps.length - 1] || started_at;
const durationMs = new Date(ended_at) - new Date(started_at);
const { skills, counts, errorCount } = collectToolUse(turn);
const { skills, counts, errors } = collectToolUse(turn);
const events = [];
for (const skill of skills) events.push({ kind: 'skill_invoked', skill });
if (Object.keys(counts).length > 0) events.push({ kind: 'tool_summary', counts });
for (let i = 0; i < errorCount; i++) {
events.push({ kind: 'error', message: 'tool_result reported is_error' });
for (const err of errors) {
events.push({ kind: 'error', tool: err.tool, summary: err.summary });
}
events.push(...extractProcessEvents(turn, broken, total, durationMs));
events.push(...extractAskUserQuestionEvents(turn));
tools/observer-transcript-parser.test.mjs
+69 -1
View File
@@ -172,7 +172,10 @@ describe('parseTranscript', () => {
assistantTurn([{ type: 'tool_use', id: 't1', name: 'Bash', input: {} }], '2026-05-19T10:01:00Z'),
toolResult('t1', true, '2026-05-19T10:02:00Z'),
]);
expect(parseTranscript(t).events.filter((e) => e.kind === 'error')).toHaveLength(1);
const errs = parseTranscript(t).events.filter((e) => e.kind === 'error');
expect(errs).toHaveLength(1);
expect(errs[0].tool).toBe('Bash');
expect(errs[0].summary).toBe('r');
});
it('skips broken JSONL lines without throwing', () => {
@@ -1180,3 +1183,68 @@ describe('parseTranscript — heuristic primary_rationale (Task 6)', () => {
expect(ep.primary_rationale.boundaries_applied).toContain('ADR-011');
});
});
describe('error event differentiation (Task 7)', () => {
it('attributes error to the failing tool and captures summary', () => {
const transcript = [
JSON.stringify({ sessionId: 's' }),
JSON.stringify({ type: 'user', message: { role: 'user', content: 'делай' }, uuid: 'u1', timestamp: '2026-05-20T00:00:00Z' }),
JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: [
{ type: 'tool_use', id: 'tu1', name: 'Bash', input: {} }
] }, uuid: 'u2', timestamp: '2026-05-20T00:00:01Z' }),
JSON.stringify({ type: 'user', message: { role: 'user', content: [
{ type: 'tool_result', tool_use_id: 'tu1', is_error: true, content: 'ENOENT: no such file or directory' }
] }, uuid: 'u3', timestamp: '2026-05-20T00:00:02Z' }),
].join('\n');
const ep = parseTranscript(transcript);
const errorEvents = ep.events.filter((e) => e.kind === 'error');
expect(errorEvents).toHaveLength(1);
expect(errorEvents[0].tool).toBe('Bash');
expect(errorEvents[0].summary).toContain('ENOENT');
});
it('attributes error to "unknown" when tool_use_id not seen', () => {
const transcript = [
JSON.stringify({ sessionId: 's' }),
JSON.stringify({ type: 'user', message: { role: 'user', content: 'делай' }, uuid: 'u1', timestamp: '2026-05-20T00:00:00Z' }),
JSON.stringify({ type: 'user', message: { role: 'user', content: [
{ type: 'tool_result', tool_use_id: 'orphan', is_error: true, content: 'mystery error' }
] }, uuid: 'u2', timestamp: '2026-05-20T00:00:02Z' }),
].join('\n');
const ep = parseTranscript(transcript);
const errs = ep.events.filter((e) => e.kind === 'error');
expect(errs[0].tool).toBe('unknown');
expect(errs[0].summary).toBe('mystery error');
});
it('truncates summary at 80 chars', () => {
const longText = 'a'.repeat(200);
const transcript = [
JSON.stringify({ sessionId: 's' }),
JSON.stringify({ type: 'user', message: { role: 'user', content: 'делай' }, uuid: 'u1', timestamp: '2026-05-20T00:00:00Z' }),
JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: [
{ type: 'tool_use', id: 'tu1', name: 'Read', input: {} }
] }, uuid: 'u2', timestamp: '2026-05-20T00:00:01Z' }),
JSON.stringify({ type: 'user', message: { role: 'user', content: [
{ type: 'tool_result', tool_use_id: 'tu1', is_error: true, content: longText }
] }, uuid: 'u3', timestamp: '2026-05-20T00:00:02Z' }),
].join('\n');
const ep = parseTranscript(transcript);
const e = ep.events.find((x) => x.kind === 'error');
expect(e.summary.length).toBe(80);
});
it('handles structured content array (text blocks)', () => {
const transcript = [
JSON.stringify({ sessionId: 's' }),
JSON.stringify({ type: 'user', message: { role: 'user', content: 'делай' }, uuid: 'u1', timestamp: '2026-05-20T00:00:00Z' }),
JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: [
{ type: 'tool_use', id: 'tu1', name: 'Edit', input: {} }
] }, uuid: 'u2', timestamp: '2026-05-20T00:00:01Z' }),
JSON.stringify({ type: 'user', message: { role: 'user', content: [
{ type: 'tool_result', tool_use_id: 'tu1', is_error: true, content: [{ type: 'text', text: 'String to replace not found' }] }
] }, uuid: 'u3', timestamp: '2026-05-20T00:00:02Z' }),
].join('\n');
const ep = parseTranscript(transcript);
const e = ep.events.find((x) => x.kind === 'error');
expect(e.tool).toBe('Edit');
expect(e.summary).toContain('String to replace');
});
});