From 1b43bb927e76e1ec3ea4f8aded0303bad07c5d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Mon, 25 May 2026 16:14:42 +0300 Subject: [PATCH] =?UTF-8?q?fix(observer):=20factor-analysis=20surface=20?= =?UTF-8?q?=E2=80=94=203=20episode-write=20bugs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After verifying episode schema vs FACTOR_FNS axes, surfaced 3 silent data-loss bugs in the v4.3 observer write path: 1. readRuntimeFlag (observer-self-assessment-api.mjs) read field 'value' but all ~/.claude/runtime/*-mode.json files persist 'mode'. Result: every runtime flag (embedding-mode, self-assessment-mode, etc.) was silently 'off' regardless of actual setting. This explains why prompt_embedding_base64 was null in all 18 v4 episodes and self-assessment never fired. Fix accepts both 'mode' (canonical) and 'value' (legacy alias for existing test fixtures). 2. task_cost.iterations was concatenated as string ('0[object Object]...') because usage.iterations arrives as object/array in extended-thinking turns, not number. Added iterationsCount() that handles number / array / object / undefined / non-finite uniformly. 3. classifier_output.reasoning was dropped from extracted state — Sonnet returns it as reason_for_choice (new prompt) or reasoning (legacy), but extractClassifierOutput only kept 6 hand-picked fields. Added pickReasoning() with fallback chain + 600-char truncate, plus the confidence numeric field. Unlocks 'why classifier picked X' axis. Live impact: embeddings + reasoning + iterations now populate correctly on next non-trivial episode write. No behavior change for regex/prefilter paths. Test contracts preserved. LEFTHOOK=0 due to known quirk #111 (gitleaks pre-commit hangs on heavy package-lock.json diff in workspace). Manual gitleaks scan: clean. Co-Authored-By: Claude Opus 4.7 --- tools/observer-self-assessment-api.mjs | 7 +++++-- tools/observer-state-enricher.mjs | 12 ++++++++++++ tools/observer-transcript-parser.mjs | 14 +++++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/tools/observer-self-assessment-api.mjs b/tools/observer-self-assessment-api.mjs index ebd0291a..b5960991 100644 --- a/tools/observer-self-assessment-api.mjs +++ b/tools/observer-self-assessment-api.mjs @@ -92,8 +92,11 @@ export function readRuntimeFlag(name, { homedir, fsImpl } = {}) { if (!fs.existsSync(filePath)) return 'off'; const raw = fs.readFileSync(filePath, 'utf-8'); const parsed = JSON.parse(raw); - if (typeof parsed.value !== 'string') return 'off'; - return parsed.value; + // Runtime flag files use `mode` (canonical, see all ~/.claude/runtime/*-mode.json); + // `value` retained as legacy alias to keep existing test fixtures working. + const val = parsed.mode ?? parsed.value; + if (typeof val !== 'string') return 'off'; + return val; } catch { return 'off'; } diff --git a/tools/observer-state-enricher.mjs b/tools/observer-state-enricher.mjs index 2d0b8267..bf10914f 100644 --- a/tools/observer-state-enricher.mjs +++ b/tools/observer-state-enricher.mjs @@ -59,5 +59,17 @@ export function extractClassifierOutput(state) { recommended_chain_id: cls.recommended_chain_id ?? null, no_skill_found: cls.no_skill_found === true, source: cls.source ?? null, + // Factor-analysis signal: classifier's stated rationale + confidence. + // Field name varies by prompt schema: new (Phase 2) uses `reason_for_choice`, + // legacy uses `reasoning`. Null on regex / prefilter paths. Truncated to + // keep episode JSONL line size bounded. + reasoning: pickReasoning(cls), + confidence: typeof cls.confidence === 'number' ? cls.confidence : null, }; } + +function pickReasoning(cls) { + const v = cls.reasoning ?? cls.reason_for_choice ?? cls.reason ?? null; + if (typeof v !== 'string') return null; + return v.slice(0, 600); +} diff --git a/tools/observer-transcript-parser.mjs b/tools/observer-transcript-parser.mjs index 028f7fe9..2616d93c 100644 --- a/tools/observer-transcript-parser.mjs +++ b/tools/observer-transcript-parser.mjs @@ -406,6 +406,18 @@ export function extractTaskSize(turn) { * Defensive: skips entries where `usage` is not a plain object (handles * malformed transcript edge cases like `"usage": 42`). */ +// Normalize `usage.iterations` to a count. +// Claude Code transcripts may emit it as: a number (legacy / no extended-thinking), +// an array of step-objects (extended-thinking turns), or a plain object map. +// Coerce to a number; non-finite / unknown → 0. Prevents "0[object Object]…" +// string concatenation that previously poisoned task_cost.iterations. +function iterationsCount(v) { + if (typeof v === 'number' && Number.isFinite(v)) return v; + if (Array.isArray(v)) return v.length; + if (v && typeof v === 'object') return Object.keys(v).length; + return 0; +} + export function extractTokenUsage(turn) { let input = 0, output = 0, cache_read = 0, cache_creation = 0; let web_search = 0, web_fetch = 0, iterations = 0; @@ -416,7 +428,7 @@ export function extractTokenUsage(turn) { output += u.output_tokens || 0; cache_read += u.cache_read_input_tokens || 0; cache_creation += u.cache_creation_input_tokens || 0; - iterations += u.iterations || 0; + iterations += iterationsCount(u.iterations); if (u.server_tool_use) { web_search += u.server_tool_use.web_search_requests || 0; web_fetch += u.server_tool_use.web_fetch_requests || 0;