From c55e14b626730c9d88cc208d54aa3bfeee4f6989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Sun, 31 May 2026 19:05:20 +0300 Subject: [PATCH] feat(brain): surface router-gate v4 signals into episode + factor axes Co-Authored-By: Claude Opus 4.8 noreply@anthropic.com --- docs/observer/brain-data-catalog.md | 16 +- ...5-31-brain-factor-analysis-f-candidates.md | 624 ++++++++++++++++++ tools/brain-retro-analyzer.mjs | 12 + tools/brain-retro-analyzer.test.mjs | 17 + tools/cost-aggregator.mjs | 10 +- tools/cost-aggregator.test.mjs | 10 +- tools/cost-pricing.mjs | 6 + tools/cost-pricing.test.mjs | 8 + tools/enforce-hook-helpers.mjs | 46 ++ tools/enforce-hook-helpers.test.mjs | 37 ++ tools/enforce-llm-judge-per-tool.mjs | 9 +- tools/enforce-llm-judge-per-tool.test.mjs | 17 + tools/enforce-safe-baseline-metering.mjs | 2 + tools/enforce-safe-baseline-metering.test.mjs | 15 + tools/observer-transcript-parser.mjs | 17 +- tools/observer-transcript-parser.test.mjs | 26 +- tools/observer-v4-signals.mjs | 74 +++ tools/observer-v4-signals.test.mjs | 49 ++ 18 files changed, 979 insertions(+), 16 deletions(-) create mode 100644 docs/superpowers/plans/2026-05-31-brain-factor-analysis-f-candidates.md create mode 100644 tools/observer-v4-signals.mjs create mode 100644 tools/observer-v4-signals.test.mjs diff --git a/docs/observer/brain-data-catalog.md b/docs/observer/brain-data-catalog.md index d53d9dcd..0705e9d7 100644 --- a/docs/observer/brain-data-catalog.md +++ b/docs/observer/brain-data-catalog.md @@ -6,7 +6,7 @@ --- -## A. Эпизод журнала — `docs/observer/episodes-YYYY-MM.jsonl` (схема v4.3) +## A. Эпизод журнала — `docs/observer/episodes-YYYY-MM.jsonl` (схема v4.4, schema_minor 4) Один эпизод = один цикл «промпт заказчика → ответ Claude». Append-only, по строке на эпизод. ПДн вырезаются до записи (`observer-pii-filter.mjs`). Сборка — `observer-transcript-parser.mjs:888` (`parseTranscript`). @@ -53,7 +53,7 @@ ### A.7. Стоимость и токены (`task_cost`) `observer-transcript-parser.mjs:472`. Базовые: `input_tokens` · `output_tokens` · `cache_read_input_tokens` · `cache_creation_input_tokens` · `web_search_requests` · `web_fetch_requests` · `iterations` (детектор extended-thinking). -Слой LLM-агентов (дозаполняется): `classifier_input_tokens` · `classifier_output_tokens` · `self_assessment_input_tokens` · `self_assessment_output_tokens` · `reviewer_input_tokens` · `reviewer_output_tokens` · `reviewer_subagent_usd` · `reviewer_direct_fallback_usd`. +Слой LLM-агентов (дозаполняется): `classifier_input_tokens` · `classifier_output_tokens` · `self_assessment_input_tokens` · `self_assessment_output_tokens` · `reviewer_input_tokens` · `reviewer_output_tokens` · `reviewer_subagent_usd` · `reviewer_direct_fallback_usd` · `judge_spend_usd` (✅ NEW — `judge_calls × JUDGE_PER_CALL_USD`). ### A.8. Мета (`task_meta`) `prompt_length_chars` · `mcp_servers_used[]` · `file_type_distribution` по корзинам `src / test / config / spec / norm / data / other` (`classifyFilePath` — `observer-transcript-parser.mjs:358`). @@ -69,6 +69,9 @@ ### A.11. События (`events[]`) — 11 видов `skill_invoked` (skill) · `tool_summary` (counts) · `error` (tool, summary) · `hook_fired` (counts, scripts, errors) · `interrupt` · `retry` · `time_burn` (ход > 15 мин) · `parse_gap` (> 10% битых строк) · `unrecovered_error` (ход кончился на ошибке) · `ask_user_question` (question_count, answer_kind: `option`/`custom`/`no_answer`) · `subagent_invoked` (subagent_type, model, description). +### A.12. Сигналы защит router-gate v4 (`v4_signals`) — ✅ NEW (schema_minor 4) +Поднимаются в эпизод ридером `observer-v4-signals.mjs` по окну хода `[started_at, ended_at]`: `rationalization_flag_count` (число пойманных самооправданий в окне) · `judge_verdict` (`YES`/`NO`/`block`/`null` — последний вердикт судьи в окне) · `safe_baseline_action` (`allow`/`soft_flag`/`hard_block`/`null` — худшее действие safe-baseline в окне) · `judge_calls` (кумулятивно за сессию из бюджета судьи). + --- ## B. Факторные оси — `FACTOR_FNS` (`brain-retro-analyzer.mjs:326`) — 27 осей + `chain_ref` @@ -85,6 +88,8 @@ **Pass 4 (семантика):** `similar_past_outcome_majority` (исход большинства соседей по смысловому вектору; `no_neighbors` если вектора нет). +**Pass 5 (✅ NEW — сигналы router-gate v4):** `rationalization_flag_count` (0/1/2+) · `judge_verdict` (YES/NO/block/null) · `safe_baseline_action` (allow/soft_flag/hard_block/null) · `judge_calls_bucket` (0 / 1-9 / 10+). + **Отдельно:** `chain_ref` — мульти-значный (эпизод считается по разу на каждую цепочку). --- @@ -133,7 +138,9 @@ Sonnet — $3/Mtok вход, $15/Mtok выход. Opus — $15/Mtok вход, $7 --- -## F. Сигналы новых защит router-gate v4 (в эпизоде ещё НЕ агрегируются — кандидаты в факторный анализ) +## F. Сигналы новых защит router-gate v4 + +> **Обновление 31.05.2026:** F.1–F.3 ✅ **заведены в эпизод** (`v4_signals`, см. A.12) и в факторный анализ (Pass 5, раздел B) через ридер `observer-v4-signals.mjs`. F.4/F.5 — пока только на диске. ### F.1. Rationalization-audit (`enforce-rationalization-audit.mjs`) `~/.claude/runtime/rationalization-flags-.jsonl`. Строка: `{kind, evidence}`. Виды (`kind`): `rationalization-phrase` (фраза-самооправдание) · `prod-edit-without-test` (правка прод-кода без теста в том же ходе) · `weak-commit-message` (commit с сообщением < 12 симв.). Блокирует на 3-м флаге за сессию (`decide:112`). Перед сопоставлением снимает цитаты/код (`stripQuotedContext:51`). @@ -153,7 +160,8 @@ Sonnet — $3/Mtok вход, $15/Mtok выход. Opus — $15/Mtok вход, $7 --- ## Кандидаты на расширение факторного анализа -Сигналы из раздела F пишутся на диск, но **не сводятся в эпизод** и не попадают в `FACTOR_FNS`. Естественные новые оси: + +✅ **Реализовано 31.05.2026** (план `docs/superpowers/plans/2026-05-31-brain-factor-analysis-f-candidates.md`): следующие 4 оси заведены в эпизод (`v4_signals`) и в `FACTOR_FNS` (Pass 5): - `rationalization_flag_count` (F.1) — число самооправданий за ход/сессию. - `safe_baseline_action` (F.2) — allow / soft_flag / hard_block. - `judge_verdict` (F.3) — YES / block / disabled (когда Layer 4 активен). diff --git a/docs/superpowers/plans/2026-05-31-brain-factor-analysis-f-candidates.md b/docs/superpowers/plans/2026-05-31-brain-factor-analysis-f-candidates.md new file mode 100644 index 00000000..e605416e --- /dev/null +++ b/docs/superpowers/plans/2026-05-31-brain-factor-analysis-f-candidates.md @@ -0,0 +1,624 @@ +# Расширение факторного анализа «мозга» — кандидаты раздела F + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Поднять сигналы новых защит router-gate v4 (rationalization-флаги, safe-baseline action, вердикт LLM-судьи, активность судьи) в эпизод журнала наблюдателя и добавить их как новые оси факторного анализа. + +**Architecture:** Сигналы уже пишутся хуками на диск в `~/.claude/runtime/`, но (1) часть не логируется в нужной форме — добавляем точечную инструментовку (verdict-log, action-log); (2) парсер транскрипта их не читает — добавляем чистый модуль-ридер `observer-v4-signals.mjs`, который `parseTranscript` зовёт и кладёт результат в новые поля эпизода; (3) анализатор не раскладывает их — добавляем 4 оси в `FACTOR_FNS`. Принцип: эпизод остаётся единственным входом факторной матрицы (хуки → runtime-файлы → парсер → эпизод → `FACTOR_FNS`). + +**Tech Stack:** Node ESM (`.mjs`), vitest (tools-only). Деньги — `bcmath`-free, float USD как в `cost-aggregator.mjs`. + +**Канонический прогон тестов tools** (per memory `feedback_vitest_sentinel_recipe`): `npm run test:tools` сломан keytar-установкой в `app/node_modules` → запускать `npx vitest run --root app --config vitest.config.tools.mjs <путь>`. + +**Базовые факты, на которых стоит план (сверены по коду 31.05.2026):** +- Эпизод собирается в `tools/observer-transcript-parser.mjs:888` (`parseTranscript`), окно хода — `turn = entries.slice(start)`, `started_at`/`ended_at` из `timestamps`. +- rationalization-флаги: `tools/enforce-hook-helpers.mjs:280` пишет `{ts, kind, evidence}` в `rationalization-flags-.jsonl`; читает `:291`. +- safe-baseline: `tools/enforce-safe-baseline-metering.mjs` пишет ledger `safe-baseline-ledger-.json` (`saveLedger:180`) и логирует ТОЛЬКО `soft_flag` в `safe-baseline-flags-.jsonl` (`logFlag:186`, строка `{ts, tool, reason}`); `hard_block` нигде не логируется. Ledger-state (`newCounterState` `safe-baseline-metering.mjs:60`) содержит `counts`/`skill_match_within_task`, но `warnings_issued`/`hard_blocks_issued` живой `runLiveDecision` НЕ заполняет. +- LLM-судья: `tools/enforce-llm-judge-per-tool.mjs:112` `runPerTool` зовёт `decide()` → вердикт `{block, verdict, degraded}`; бюджет `tools/llm-judge.mjs:121` `readJudgeBudget`/`:127` `bumpJudgeBudget` хранит `{calls}` в `llm-judge-budget-.json`; вердикт по-инструментно НЕ логируется. `JUDGE_SESSION_BUDGET = 200` (`llm-judge.mjs:67`). +- Цена: `tools/cost-pricing.mjs` `PRICING.sonnet = {input:3e-6, output:15e-6}`; агрегатор `tools/cost-aggregator.mjs:13` `episodeUsd` / `:40` `aggregateDay` (5 компонент + `total_usd` + `episode_count`); Stop-хук `tools/cost-stop-hook.mjs` пишет `cost-daily.json`. +- Факторные оси: `tools/brain-retro-analyzer.mjs:326` `FACTOR_FNS`; `analyze()` `:657` штампует `episode._inferredOutcome` и строит матрицу `buildFactorMatrix:384`. + +--- + +## File Structure + +| Файл | Ответственность | Действие | +|---|---|---| +| `tools/enforce-llm-judge-per-tool.mjs` | вызов судьи per-tool | Modify — логировать вердикт | +| `tools/enforce-hook-helpers.mjs` | общий I/O хуков | Modify — `logJudgeVerdict` + `readJudgeVerdicts` + `logSafeBaselineAction` + `readSafeBaselineActions` | +| `tools/enforce-safe-baseline-metering.mjs` | safe-baseline метеринг | Modify — логировать ЛЮБОЙ action (incl hard_block) | +| `tools/observer-v4-signals.mjs` | **новый** чистый ридер runtime-сигналов в окне хода | Create | +| `tools/observer-transcript-parser.mjs` | сборка эпизода | Modify — позвать ридер, добавить поля эпизода | +| `tools/cost-pricing.mjs` | ставки | Modify — `JUDGE_PER_CALL_USD` | +| `tools/cost-aggregator.mjs` | стоимость per-episode/day | Modify — компонент `judge_spend_usd` | +| `tools/brain-retro-analyzer.mjs` | факторные оси | Modify — 4 новые `FACTOR_FNS` | +| `docs/observer/brain-data-catalog.md` | каталог данных мозга | Modify — перенести сигналы из F в A/B/E | +| `tools/*.test.mjs` | тесты | Create/Modify per task | + +**Новые поля эпизода (schema_minor bump 3 → 4):** +- `task_cost.judge_spend_usd` (number, default 0) +- `v4_signals.rationalization_flag_count` (int) +- `v4_signals.safe_baseline_action` (`allow`/`soft_flag`/`hard_block`/`null`) +- `v4_signals.judge_verdict` (`YES`/`NO`/`block`/`disabled`/`null`) +- `v4_signals.judge_calls` (int) + +--- + +## Task 1: Логирование вердикта судьи + +**Files:** +- Modify: `tools/enforce-hook-helpers.mjs` (после `readRationalizationFlags`, ~:299) +- Modify: `tools/enforce-llm-judge-per-tool.mjs` (`runPerTool`, :112) +- Test: `tools/enforce-hook-helpers.test.mjs` + +- [ ] **Step 1: Написать падающий тест на `logJudgeVerdict`/`readJudgeVerdicts`** + +```js +// tools/enforce-hook-helpers.test.mjs — добавить describe-блок +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { logJudgeVerdict, readJudgeVerdicts } from './enforce-hook-helpers.mjs'; + +describe('judge verdict log', () => { + it('appends and reads back verdict records', () => { + const dir = mkdtempSync(join(tmpdir(), 'jv-')); + logJudgeVerdict('sess1', { tool: 'Edit', verdict: 'YES' }, { baseDir: dir }); + logJudgeVerdict('sess1', { tool: 'Bash', verdict: 'NO' }, { baseDir: dir }); + const recs = readJudgeVerdicts('sess1', { baseDir: dir }); + expect(recs.length).toBe(2); + expect(recs[0]).toMatchObject({ tool: 'Edit', verdict: 'YES' }); + expect(typeof recs[0].ts).toBe('string'); + rmSync(dir, { recursive: true, force: true }); + }); + it('returns [] when no file', () => { + expect(readJudgeVerdicts('nope', { baseDir: tmpdir() })).toEqual([]); + }); +}); +``` + +- [ ] **Step 2: Прогнать — убедиться, что падает** + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/enforce-hook-helpers.test.mjs` +Expected: FAIL — `logJudgeVerdict is not a function`. + +- [ ] **Step 3: Реализовать функции в `enforce-hook-helpers.mjs`** (вставить после `readRationalizationFlags`, перед `readRouterState`) + +```js +export function logJudgeVerdict(sessionId, { tool, verdict }, opts = {}) { + try { + const dir = opts.baseDir || runtimeDir(); + const f = join(dir, `llm-judge-verdicts-${sessionId || 'unknown'}.jsonl`); + appendFileSync(f, JSON.stringify({ + ts: new Date().toISOString(), + tool: tool || 'unknown', + verdict: verdict == null ? null : String(verdict), + }) + '\n'); + } catch { /* ignore */ } +} + +export function readJudgeVerdicts(sessionId, opts = {}) { + try { + const dir = opts.baseDir || runtimeDir(); + const f = join(dir, `llm-judge-verdicts-${sessionId || 'unknown'}.jsonl`); + if (!existsSync(f)) return []; + return readFileSync(f, 'utf-8').split('\n').filter(Boolean).map((l) => { + try { return JSON.parse(l); } catch { return null; } + }).filter(Boolean); + } catch { return []; } +} +``` + +(`runtimeDir`, `appendFileSync`, `readFileSync`, `existsSync`, `join` уже импортированы в файле — проверить шапку, при отсутствии `existsSync` добавить в существующий `import { ... } from 'node:fs'`.) + +- [ ] **Step 4: Прогнать тест — PASS** + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/enforce-hook-helpers.test.mjs` +Expected: PASS. + +- [ ] **Step 5: Вызвать логгер из судьи.** В `enforce-llm-judge-per-tool.mjs:112` `runPerTool`, после строки `if (result.verdict !== undefined) bumpBudgetImpl({ sessionId, by: 1 });` добавить: + +```js + if (result.verdict !== undefined) { + bumpBudgetImpl({ sessionId, by: 1 }); + logVerdictImpl({ sessionId, tool: event && event.tool_name, verdict: result.block ? 'block' : (result.verdict ?? 'YES') }); + } +``` + +В сигнатуру `runPerTool({...})` добавить параметр `logVerdictImpl = () => {}` (no-op по умолчанию — тесты не пишут на диск). В `main()` (:153) передать `logVerdictImpl: ({ sessionId, tool, verdict }) => logJudgeVerdict(sessionId, { tool, verdict })` и добавить `logJudgeVerdict` в импорт из `./enforce-hook-helpers.mjs`. + +- [ ] **Step 6: Тест на проброс `logVerdictImpl`** + +```js +// tools/enforce-llm-judge-per-tool.test.mjs — добавить +it('runPerTool logs verdict only when a real judge call happened', async () => { + const logged = []; + const r = await runPerTool({ + event: { tool_name: 'Edit', session_id: 's', tool_input: {} }, + judgeConfig: { enabled: true, apiKey: 'k' }, + readDeclaredTaskImpl: () => ({ task_summary: 't' }), + readBudgetImpl: () => 0, + bumpBudgetImpl: () => {}, + llmJudgeCallImpl: async () => 'YES', + logVerdictImpl: (rec) => logged.push(rec), + }); + expect(r.block).toBe(false); + expect(logged).toEqual([{ sessionId: 's', tool: 'Edit', verdict: 'YES' }]); +}); +``` + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/enforce-llm-judge-per-tool.test.mjs` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add tools/enforce-hook-helpers.mjs tools/enforce-hook-helpers.test.mjs tools/enforce-llm-judge-per-tool.mjs tools/enforce-llm-judge-per-tool.test.mjs +git commit -m "feat(brain): log per-tool judge verdicts to runtime jsonl" +``` + +--- + +## Task 2: Логирование любого safe-baseline action (incl hard_block) + +**Files:** +- Modify: `tools/enforce-hook-helpers.mjs` +- Modify: `tools/enforce-safe-baseline-metering.mjs` (`runMain`, :195) +- Test: `tools/enforce-hook-helpers.test.mjs`, `tools/enforce-safe-baseline-metering.test.mjs` + +- [ ] **Step 1: Тест на `logSafeBaselineAction`/`readSafeBaselineActions`** + +```js +// tools/enforce-hook-helpers.test.mjs +import { logSafeBaselineAction, readSafeBaselineActions } from './enforce-hook-helpers.mjs'; +describe('safe-baseline action log', () => { + it('appends and reads action records', () => { + const dir = mkdtempSync(join(tmpdir(), 'sb-')); + logSafeBaselineAction('s', { tool: 'Edit', action: 'hard_block' }, { baseDir: dir }); + const recs = readSafeBaselineActions('s', { baseDir: dir }); + expect(recs[0]).toMatchObject({ tool: 'Edit', action: 'hard_block' }); + rmSync(dir, { recursive: true, force: true }); + }); +}); +``` + +- [ ] **Step 2: Прогнать — FAIL** (`logSafeBaselineAction is not a function`). + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/enforce-hook-helpers.test.mjs` + +- [ ] **Step 3: Реализовать** (рядом с judge-логгером из Task 1) + +```js +export function logSafeBaselineAction(sessionId, { tool, action }, opts = {}) { + try { + const dir = opts.baseDir || runtimeDir(); + const f = join(dir, `safe-baseline-actions-${sessionId || 'unknown'}.jsonl`); + appendFileSync(f, JSON.stringify({ + ts: new Date().toISOString(), + tool: tool || 'unknown', + action: String(action || 'allow'), + }) + '\n'); + } catch { /* ignore */ } +} + +export function readSafeBaselineActions(sessionId, opts = {}) { + try { + const dir = opts.baseDir || runtimeDir(); + const f = join(dir, `safe-baseline-actions-${sessionId || 'unknown'}.jsonl`); + if (!existsSync(f)) return []; + return readFileSync(f, 'utf-8').split('\n').filter(Boolean).map((l) => { + try { return JSON.parse(l); } catch { return null; } + }).filter(Boolean); + } catch { return []; } +} +``` + +- [ ] **Step 4: Прогнать — PASS.** + +- [ ] **Step 5: Вызвать из `enforce-safe-baseline-metering.mjs` `runMain`.** Заменить блок `if (res.action === 'soft_flag') logFlag(...)` на запись любого не-allow действия через новый логгер, сохранив старый `logFlag` для совместимости: + +```js + if (res.action !== 'allow') { + logSafeBaselineAction(sess, { tool: event.tool_name, action: res.action }); + if (res.action === 'soft_flag') logFlag(dir, sess, { tool: event.tool_name, reason: res.reason }); + } +``` + +Добавить `import { logSafeBaselineAction } from './enforce-hook-helpers.mjs';` (или дополнить существующий импорт из helpers). NB: `logSafeBaselineAction` сам резолвит `runtimeDir()` — в проде это совпадает с `dir`; для теста `runMain` принимает `runtimeDir` override, но логгер пишет в реальный runtime — поэтому в тесте инъектировать override не обязательно (проверяем экспортируемый логгер отдельно в Step 1). + +- [ ] **Step 6: Регрессия safe-baseline.** + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/enforce-safe-baseline-metering.test.mjs` +Expected: PASS (существующие тесты не трогают `action !== 'allow'` ветку с диском, либо обновить ожидания). + +- [ ] **Step 7: Commit** + +```bash +git add tools/enforce-hook-helpers.mjs tools/enforce-hook-helpers.test.mjs tools/enforce-safe-baseline-metering.mjs tools/enforce-safe-baseline-metering.test.mjs +git commit -m "feat(brain): log every safe-baseline action incl hard_block" +``` + +--- + +## Task 3: Чистый ридер сигналов в окне хода — `observer-v4-signals.mjs` + +**Files:** +- Create: `tools/observer-v4-signals.mjs` +- Test: `tools/observer-v4-signals.test.mjs` + +- [ ] **Step 1: Тест ридера** + +```js +// tools/observer-v4-signals.test.mjs +import { describe, it, expect } from 'vitest'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { extractV4Signals } from './observer-v4-signals.mjs'; + +function write(dir, name, lines) { writeFileSync(join(dir, name), lines.map((l) => JSON.stringify(l)).join('\n') + '\n'); } + +describe('extractV4Signals', () => { + it('counts rationalization flags inside the turn window only', () => { + const dir = mkdtempSync(join(tmpdir(), 'v4-')); + write(dir, 'rationalization-flags-s.jsonl', [ + { ts: '2026-05-31T10:00:00.000Z', kind: 'x' }, // before window + { ts: '2026-05-31T10:05:00.000Z', kind: 'y' }, // in + { ts: '2026-05-31T10:06:00.000Z', kind: 'z' }, // in + ]); + write(dir, 'llm-judge-verdicts-s.jsonl', [ + { ts: '2026-05-31T10:05:30.000Z', tool: 'Edit', verdict: 'YES' }, + { ts: '2026-05-31T10:05:40.000Z', tool: 'Bash', verdict: 'block' }, + ]); + write(dir, 'safe-baseline-actions-s.jsonl', [ + { ts: '2026-05-31T10:05:10.000Z', tool: 'Edit', action: 'soft_flag' }, + { ts: '2026-05-31T10:05:50.000Z', tool: 'Write', action: 'hard_block' }, + ]); + writeFileSync(join(dir, 'llm-judge-budget-s.json'), JSON.stringify({ calls: 7 })); + const sig = extractV4Signals('s', { + startMs: Date.parse('2026-05-31T10:04:00.000Z'), + endMs: Date.parse('2026-05-31T10:07:00.000Z'), + baseDir: dir, + }); + expect(sig.rationalization_flag_count).toBe(2); + expect(sig.judge_verdict).toBe('block'); // last in-window verdict + expect(sig.safe_baseline_action).toBe('hard_block'); // worst in-window action + expect(sig.judge_calls).toBe(7); + rmSync(dir, { recursive: true, force: true }); + }); + it('returns zero/null defaults when files absent', () => { + const sig = extractV4Signals('nope', { startMs: 0, endMs: 1, baseDir: tmpdir() }); + expect(sig).toEqual({ rationalization_flag_count: 0, judge_verdict: null, safe_baseline_action: null, judge_calls: 0 }); + }); +}); +``` + +- [ ] **Step 2: Прогнать — FAIL** (модуль не существует). + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/observer-v4-signals.test.mjs` + +- [ ] **Step 3: Реализовать `observer-v4-signals.mjs`** + +```js +#!/usr/bin/env node +/** + * Pure reader of router-gate v4 runtime signals, scoped to one episode's + * [startMs, endMs] turn window. Feeds observer-transcript-parser (new + * episode.v4_signals block + task_cost.judge_spend_usd). No exec/exit. + */ +import { readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { homedir } from 'node:os'; + +function rtDir(override) { return override || join(homedir(), '.claude', 'runtime'); } + +function readJsonl(path) { + if (!existsSync(path)) return []; + try { + return readFileSync(path, 'utf-8').split('\n').filter(Boolean).map((l) => { + try { return JSON.parse(l); } catch { return null; } + }).filter(Boolean); + } catch { return []; } +} + +function inWindow(rec, startMs, endMs) { + const ms = Date.parse(rec && rec.ts); + return Number.isFinite(ms) && ms >= startMs && ms <= endMs; +} + +// soft order: hard_block > soft_flag > allow. +const ACTION_RANK = { allow: 0, soft_flag: 1, hard_block: 2 }; +function worstAction(records) { + let best = null; + for (const r of records) { + const a = r && r.action; + if (a == null) continue; + if (best === null || (ACTION_RANK[a] ?? -1) > (ACTION_RANK[best] ?? -1)) best = a; + } + return best; +} + +export function extractV4Signals(sessionId, { startMs, endMs, baseDir } = {}) { + const dir = rtDir(baseDir); + const sess = sessionId || 'unknown'; + + const flags = readJsonl(join(dir, `rationalization-flags-${sess}.jsonl`)) + .filter((r) => inWindow(r, startMs, endMs)); + + const verdicts = readJsonl(join(dir, `llm-judge-verdicts-${sess}.jsonl`)) + .filter((r) => inWindow(r, startMs, endMs)); + const judge_verdict = verdicts.length ? (verdicts[verdicts.length - 1].verdict ?? null) : null; + + const actions = readJsonl(join(dir, `safe-baseline-actions-${sess}.jsonl`)) + .filter((r) => inWindow(r, startMs, endMs)); + const safe_baseline_action = worstAction(actions); + + let judge_calls = 0; + const bp = join(dir, `llm-judge-budget-${sess}.json`); + if (existsSync(bp)) { + try { judge_calls = Number(JSON.parse(readFileSync(bp, 'utf-8')).calls) || 0; } catch { /* keep 0 */ } + } + + return { + rationalization_flag_count: flags.length, + judge_verdict, + safe_baseline_action, + judge_calls, + }; +} +``` + +- [ ] **Step 4: Прогнать — PASS.** + +- [ ] **Step 5: Commit** + +```bash +git add tools/observer-v4-signals.mjs tools/observer-v4-signals.test.mjs +git commit -m "feat(brain): pure reader for v4 runtime signals in turn window" +``` + +--- + +## Task 4: Внести сигналы в эпизод (`parseTranscript`) + +**Files:** +- Modify: `tools/observer-transcript-parser.mjs` (:888 return object) +- Test: `tools/observer-transcript-parser.test.mjs` + +- [ ] **Step 1: Тест — эпизод несёт `v4_signals` и `task_cost.judge_spend_usd`** + +```js +// tools/observer-transcript-parser.test.mjs — добавить +it('episode carries v4_signals block (default zero/null when no runtime files)', () => { + const ep = parseTranscript(JSON.stringify({ + sessionId: 'no-runtime-sess', + message: { role: 'user', content: 'сделай фичу X' }, + timestamp: '2026-05-31T10:00:00.000Z', + }), null, { routerStateBaseDir: tmpdir() }); + expect(ep.v4_signals).toEqual({ + rationalization_flag_count: 0, judge_verdict: null, safe_baseline_action: null, judge_calls: 0, + }); + expect(typeof ep.task_cost.judge_spend_usd).toBe('number'); + expect(ep.schema_minor).toBe(4); +}); +``` + +(`tmpdir` импортировать в тест-файле, если ещё не.) + +- [ ] **Step 2: Прогнать — FAIL** (`ep.v4_signals` undefined). + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/observer-transcript-parser.test.mjs` + +- [ ] **Step 3: Реализовать.** В `observer-transcript-parser.mjs`: + +(а) импорт вверху рядом с другими: `import { extractV4Signals } from './observer-v4-signals.mjs';` и `import { JUDGE_PER_CALL_USD } from './cost-pricing.mjs';` + +(б) в `parseTranscript`, после вычисления `started_at`/`ended_at` (:847), добавить: + +```js + const _v4 = extractV4Signals(sessionId, { + startMs: new Date(started_at).getTime(), + endMs: new Date(ended_at).getTime(), + baseDir: options.runtimeBaseDir || routerStateBaseDir, + }); +``` + +(в) bump `schema_minor: 3` → `schema_minor: 4` в return. + +(г) в return-объект добавить поле `v4_signals: _v4,`. + +(д) в `task_cost` влить судейский spend: заменить строку `task_cost: { ...extractTokenUsage(turn), ...((routerState && routerState.task_cost) || {}) },` на: + +```js + task_cost: { + ...extractTokenUsage(turn), + ...((routerState && routerState.task_cost) || {}), + judge_spend_usd: _v4.judge_calls * JUDGE_PER_CALL_USD, + }, +``` + +NB: `routerStateBaseDir` уже извлекается из `options` (:838). `extractV4Signals` принимает тот же baseDir — в проде это `~/.claude/runtime`, в тестах — temp. + +- [ ] **Step 4: Прогнать — PASS** (после Task 5, где появится `JUDGE_PER_CALL_USD`). Если выполняется до Task 5 — импорт `JUDGE_PER_CALL_USD` упадёт; **зависимость: Task 5 Step 3 делать до Task 4 Step 3** (или внести константу в этом же шаге). Для линейного исполнения: добавить `JUDGE_PER_CALL_USD` в `cost-pricing.mjs` здесь же (Step 3.е) и продублировать тест в Task 5. + +(3.е) В `tools/cost-pricing.mjs` добавить экспорт: + +```js +// Оценочная средняя стоимость одного вызова per-tool судьи (Sonnet, ~600 in / ~10 out +// токенов на короткий YES/NO-промпт). Уточнить, когда появится по-вызовный учёт токенов. +export const JUDGE_PER_CALL_USD = 600 * PRICING.sonnet.input + 10 * PRICING.sonnet.output; +``` + +- [ ] **Step 5: Прогнать целевой тест — PASS.** + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/observer-transcript-parser.test.mjs` + +- [ ] **Step 6: Commit** + +```bash +git add tools/observer-transcript-parser.mjs tools/observer-transcript-parser.test.mjs tools/cost-pricing.mjs +git commit -m "feat(brain): surface v4_signals + judge_spend_usd into episode (schema_minor 4)" +``` + +--- + +## Task 5: Денежный компонент `judge_spend_usd` в агрегаторе и cost-daily + +**Files:** +- Modify: `tools/cost-aggregator.mjs` (`episodeUsd` :13, `aggregateDay` :40) +- Test: `tools/cost-aggregator.test.mjs` + +- [ ] **Step 1: Тест — `episodeUsd` отдаёт `judge_spend_usd`, `total_usd` его включает** + +```js +// tools/cost-aggregator.test.mjs +it('includes judge_spend_usd in episodeUsd + total', () => { + const pricing = { sonnet: { input: 3e-6, output: 15e-6 }, opus: { input: 15e-6, output: 75e-6 } }; + const ep = { task_cost: { judge_spend_usd: 0.002 } }; + const u = episodeUsd(ep, pricing); + expect(u.judge_spend_usd).toBe(0.002); + expect(u.total_usd).toBeCloseTo(0.002, 9); +}); +``` + +- [ ] **Step 2: Прогнать — FAIL** (`u.judge_spend_usd` undefined). + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/cost-aggregator.test.mjs` + +- [ ] **Step 3: Реализовать.** В `episodeUsd` (:13) добавить строку и включить в `total_usd` + объект: + +```js + const judge_spend_usd = tc.judge_spend_usd || 0; + const total_usd = + classifier_usd + self_assessment_usd + reviewer_subagent_usd + + reviewer_direct_fallback_usd + self_retrospect_usd + judge_spend_usd; + return { + classifier_usd, self_assessment_usd, reviewer_subagent_usd, + reviewer_direct_fallback_usd, self_retrospect_usd, judge_spend_usd, total_usd, + }; +``` + +В `aggregateDay` (:40) добавить `judge_spend_usd: 0` в `totals`, прибавлять `totals.judge_spend_usd += u.judge_spend_usd;` в цикле и включить в финальный `total_usd`. + +- [ ] **Step 4: Прогнать — PASS** + регрессия `cost-stop-hook`: + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/cost-aggregator.test.mjs tools/cost-stop-hook.test.mjs` +Expected: PASS (cost-daily.json теперь получит поле `judge_spend_usd` автоматически — оно идёт из `aggregateDay`; при необходимости обновить ожидания в `cost-stop-hook.test.mjs`). + +- [ ] **Step 5: Commit** + +```bash +git add tools/cost-aggregator.mjs tools/cost-aggregator.test.mjs tools/cost-stop-hook.test.mjs +git commit -m "feat(brain): add judge_spend_usd cost component to aggregator + cost-daily" +``` + +--- + +## Task 6: Четыре новые факторные оси + +**Files:** +- Modify: `tools/brain-retro-analyzer.mjs` (`FACTOR_FNS` :326) +- Test: `tools/brain-retro-analyzer.test.mjs` + +- [ ] **Step 1: Тест — матрица содержит новые оси** + +```js +// tools/brain-retro-analyzer.test.mjs +import { buildFactorMatrix } from './brain-retro-analyzer.mjs'; +it('factor matrix exposes v4-signal axes', () => { + const eps = [ + { _inferredOutcome: 'success', v4_signals: { rationalization_flag_count: 0, judge_verdict: 'YES', safe_baseline_action: 'allow', judge_calls: 0 }, task_cost: { judge_spend_usd: 0 } }, + { _inferredOutcome: 'rework', v4_signals: { rationalization_flag_count: 3, judge_verdict: 'block', safe_baseline_action: 'hard_block', judge_calls: 12 }, task_cost: { judge_spend_usd: 0.03 } }, + ]; + const m = buildFactorMatrix(eps); + expect(m.rationalization_flag_count['0']).toEqual({ success: 1 }); + expect(m.rationalization_flag_count['2+']).toEqual({ rework: 1 }); + expect(m.judge_verdict.YES).toEqual({ success: 1 }); + expect(m.judge_verdict.block).toEqual({ rework: 1 }); + expect(m.safe_baseline_action.hard_block).toEqual({ rework: 1 }); + expect(m.judge_calls_bucket['0']).toEqual({ success: 1 }); + expect(m.judge_calls_bucket['10+']).toEqual({ rework: 1 }); +}); +``` + +- [ ] **Step 2: Прогнать — FAIL** (осей нет). + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/brain-retro-analyzer.test.mjs` + +- [ ] **Step 3: Реализовать.** В `FACTOR_FNS` (:326) добавить (используя существующий `countBucket012` :321 для 0/1/2+ и новый локальный bucket для judge_calls): + +```js + // Pass 5 — router-gate v4 signal axes (brain-data-catalog раздел F → факторы). + rationalization_flag_count: (e) => countBucket012((e.v4_signals || {}).rationalization_flag_count), + judge_verdict: (e) => (e.v4_signals || {}).judge_verdict || 'null', + safe_baseline_action: (e) => (e.v4_signals || {}).safe_baseline_action || 'null', + judge_calls_bucket: (e) => judgeCallsBucket((e.v4_signals || {}).judge_calls), +``` + +И рядом с другими bucket-хелперами (около :321) добавить: + +```js +function judgeCallsBucket(n) { + const v = Number(n) || 0; + if (v === 0) return '0'; + if (v < 10) return '1-9'; + return '10+'; +} +``` + +- [ ] **Step 4: Прогнать — PASS** + полная регрессия анализатора: + +Run: `npx vitest run --root app --config vitest.config.tools.mjs tools/brain-retro-analyzer.test.mjs` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add tools/brain-retro-analyzer.mjs tools/brain-retro-analyzer.test.mjs +git commit -m "feat(brain): add 4 v4-signal factor axes (rationalization/judge/safe-baseline)" +``` + +--- + +## Task 7: Обновить каталог данных + полная регрессия tools + +**Files:** +- Modify: `docs/observer/brain-data-catalog.md` + +- [ ] **Step 1: Перенести сигналы из раздела F.** В `brain-data-catalog.md`: в A.7 добавить `judge_spend_usd` в список `task_cost`; добавить новую группу **A.12 `v4_signals`** (`rationalization_flag_count`, `judge_verdict`, `safe_baseline_action`, `judge_calls`); в раздел B дописать «Pass 5 (4 оси): rationalization_flag_count / judge_verdict / safe_baseline_action / judge_calls_bucket»; в разделе F пометить F.1–F.3 как «✅ заведено в эпизод (см. A.12)», оставив F.4/F.5 как есть; bump «схема v4.3» → «v4.4 (schema_minor 4)». + +- [ ] **Step 2: Полная регрессия tools-only** + +Run: `npx vitest run --root app --config vitest.config.tools.mjs` +Expected: все GREEN (новые тесты + прежний baseline без падений). + +- [ ] **Step 3: Commit** + +```bash +git add docs/observer/brain-data-catalog.md +git commit -m "docs(observer): catalog — v4 signals promoted from F into episode + factors" +``` + +--- + +## Self-Review + +**1. Покрытие требований (кандидаты F из каталога):** +- `rationalization_flag_count` → Task 3 (ридер) + Task 4 (поле) + Task 6 (ось). ✅ +- `safe_baseline_action` → Task 2 (логирование) + Task 3 + Task 4 + Task 6. ✅ +- `judge_verdict` → Task 1 (логирование) + Task 3 + Task 4 + Task 6. ✅ +- `judge_spend_usd` → Task 4 (поле, из `judge_calls × JUDGE_PER_CALL_USD`) + Task 5 (агрегатор/cost-daily). ✅ +- `judge_calls` → Task 3 (бюджет) + Task 6 (ось `judge_calls_bucket`). ✅ + +**2. Заглушки:** код в каждом шаге полный; команды и ожидания указаны. `JUDGE_PER_CALL_USD` — явная оценочная константа с комментарием (не TODO-заглушка), уточняемая, когда появится по-вызовный учёт токенов. + +**3. Согласованность типов/имён:** +- `extractV4Signals(sessionId, {startMs,endMs,baseDir})` → возвращает `{rationalization_flag_count, judge_verdict, safe_baseline_action, judge_calls}` — те же ключи в Task 3 тесте, Task 4 поле `v4_signals`, Task 6 осях. ✅ +- `logJudgeVerdict`/`readJudgeVerdicts`, `logSafeBaselineAction`/`readSafeBaselineActions` — пары имён согласованы между Task 1/2 и Task 3 (файлы `llm-judge-verdicts-.jsonl`, `safe-baseline-actions-.jsonl`). ✅ +- `judge_spend_usd` — одно имя в Task 4 (запись), Task 5 (агрегатор), каталоге. ✅ +- **Зависимость порядка:** Task 5 Step 3 (или Task 4 Step 3.е) должен внести `JUDGE_PER_CALL_USD` в `cost-pricing.mjs` ДО импорта в `observer-transcript-parser.mjs`. Зафиксировано в Task 4 Step 4 примечанием. + +**4. Риски/допущения, проверить при исполнении:** +- `observer-transcript-parser` читает runtime по абсолютному `baseDir` (прод `~/.claude/runtime`); тесты инъектируют temp через `options.runtimeBaseDir || routerStateBaseDir`. +- `judge_calls` — кумулятивно per-session (бюджет не сбрасывается per-turn), поэтому `judge_calls_bucket` отражает нагрузку сессии, не хода; это осознанно (для USD это и нужно). Если потребуется per-turn — считать вердикты в окне (`verdicts.length`), отдельная ось. +- `enforce-llm-judge-per-tool` логирует вердикт только при реальном вызове (`result.verdict !== undefined`) — disabled/readonly/test-runner не пишут (verdict отсутствует) → `judge_verdict` останется `null`, что корректно отражает «судья не сработал». diff --git a/tools/brain-retro-analyzer.mjs b/tools/brain-retro-analyzer.mjs index 3ebefb31..27e79368 100644 --- a/tools/brain-retro-analyzer.mjs +++ b/tools/brain-retro-analyzer.mjs @@ -323,6 +323,13 @@ function countBucket012(n) { return v === 0 ? '0' : v === 1 ? '1' : '2+'; } +function judgeCallsBucket(n) { + const v = Number(n) || 0; + if (v === 0) return '0'; + if (v < 10) return '1-9'; + return '10+'; +} + const FACTOR_FNS = { decision_provenance: (e) => (e.decision_provenance || {}).kind || 'unknown', economy_level: (e) => String((e.environment || {}).economy_level ?? 'null'), @@ -362,6 +369,11 @@ const FACTOR_FNS = { // (cross-episode pass via observer-embedding-index). Episodes without an // embedding or with no resolved neighbours bucket as 'no_neighbors'. similar_past_outcome_majority: (e) => e._similarPastOutcomeMajority || 'no_neighbors', + // Pass 5 — router-gate v4 signal axes (brain-data-catalog раздел F → факторы). + rationalization_flag_count: (e) => countBucket012((e.v4_signals || {}).rationalization_flag_count), + judge_verdict: (e) => (e.v4_signals || {}).judge_verdict || 'null', + safe_baseline_action: (e) => (e.v4_signals || {}).safe_baseline_action || 'null', + judge_calls_bucket: (e) => judgeCallsBucket((e.v4_signals || {}).judge_calls), }; // Pass 4 — decode prompt_embedding_base64 to Float32Array. Mirrors diff --git a/tools/brain-retro-analyzer.test.mjs b/tools/brain-retro-analyzer.test.mjs index 7fd3b4d4..47f99038 100644 --- a/tools/brain-retro-analyzer.test.mjs +++ b/tools/brain-retro-analyzer.test.mjs @@ -431,6 +431,23 @@ describe('analyze — v4 aggregations (Phase 3 Task 20)', () => { }); }); +describe('buildFactorMatrix — Pass 5 v4-signal axes', () => { + it('factor matrix exposes v4-signal axes', () => { + const eps = [ + { _inferredOutcome: 'success', v4_signals: { rationalization_flag_count: 0, judge_verdict: 'YES', safe_baseline_action: 'allow', judge_calls: 0 }, task_cost: { judge_spend_usd: 0 } }, + { _inferredOutcome: 'rework', v4_signals: { rationalization_flag_count: 3, judge_verdict: 'block', safe_baseline_action: 'hard_block', judge_calls: 12 }, task_cost: { judge_spend_usd: 0.03 } }, + ]; + const m = buildFactorMatrix(eps); + expect(m.rationalization_flag_count['0']).toEqual({ success: 1 }); + expect(m.rationalization_flag_count['2+']).toEqual({ rework: 1 }); + expect(m.judge_verdict.YES).toEqual({ success: 1 }); + expect(m.judge_verdict.block).toEqual({ rework: 1 }); + expect(m.safe_baseline_action.hard_block).toEqual({ rework: 1 }); + expect(m.judge_calls_bucket['0']).toEqual({ success: 1 }); + expect(m.judge_calls_bucket['10+']).toEqual({ rework: 1 }); + }); +}); + describe('buildFactorMatrix — Pass 1 cheap axes (project-brain-factor-analysis-4passes)', () => { // Each new axis: smoke + null-safety on missing fields. it('prompt_signal axis: raw discrete values + null fallback', () => { diff --git a/tools/cost-aggregator.mjs b/tools/cost-aggregator.mjs index a37974c3..f2e9db3f 100644 --- a/tools/cost-aggregator.mjs +++ b/tools/cost-aggregator.mjs @@ -21,18 +21,21 @@ export function episodeUsd(episode, pricing) { const reviewer_subagent_usd = tc.reviewer_subagent_usd || 0; const reviewer_direct_fallback_usd = tc.reviewer_direct_fallback_usd || 0; const self_retrospect_usd = tc.self_retrospect_usd || 0; + const judge_spend_usd = tc.judge_spend_usd || 0; const total_usd = classifier_usd + self_assessment_usd + reviewer_subagent_usd + reviewer_direct_fallback_usd + - self_retrospect_usd; + self_retrospect_usd + + judge_spend_usd; return { classifier_usd, self_assessment_usd, reviewer_subagent_usd, reviewer_direct_fallback_usd, self_retrospect_usd, + judge_spend_usd, total_usd, }; } @@ -44,6 +47,7 @@ export function aggregateDay(episodes, dateISO, pricing) { reviewer_subagent_usd: 0, reviewer_direct_fallback_usd: 0, self_retrospect_usd: 0, + judge_spend_usd: 0, }; let episode_count = 0; for (const ep of Array.isArray(episodes) ? episodes : []) { @@ -55,6 +59,7 @@ export function aggregateDay(episodes, dateISO, pricing) { totals.reviewer_subagent_usd += u.reviewer_subagent_usd; totals.reviewer_direct_fallback_usd += u.reviewer_direct_fallback_usd; totals.self_retrospect_usd += u.self_retrospect_usd; + totals.judge_spend_usd += u.judge_spend_usd; episode_count += 1; } const total_usd = @@ -62,6 +67,7 @@ export function aggregateDay(episodes, dateISO, pricing) { totals.self_assessment_usd + totals.reviewer_subagent_usd + totals.reviewer_direct_fallback_usd + - totals.self_retrospect_usd; + totals.self_retrospect_usd + + totals.judge_spend_usd; return { ...totals, total_usd, episode_count }; } diff --git a/tools/cost-aggregator.test.mjs b/tools/cost-aggregator.test.mjs index e70fccb7..be0b3f90 100644 --- a/tools/cost-aggregator.test.mjs +++ b/tools/cost-aggregator.test.mjs @@ -13,6 +13,13 @@ describe('episodeUsd', () => { expect(u.total_usd).toBe(0); }); + it('includes judge_spend_usd in episodeUsd + total', () => { + const ep = { task_cost: { judge_spend_usd: 0.002 } }; + const u = episodeUsd(ep, PRICING); + expect(u.judge_spend_usd).toBe(0.002); + expect(u.total_usd).toBeCloseTo(0.002, 9); + }); + it('computes classifier_usd from sonnet pricing', () => { const ep = { task_cost: { classifier_input_tokens: 1_000_000, classifier_output_tokens: 100_000 } }; const u = episodeUsd(ep, PRICING); @@ -96,11 +103,12 @@ describe('aggregateDay', () => { expect(r.classifier_usd).toBeCloseTo(3, 6); }); - it('returns object with exactly 7 keys (5 components + total + count)', () => { + it('returns object with exactly 8 keys (6 components + total + count)', () => { const r = aggregateDay([], '2026-05-28', PRICING); expect(Object.keys(r).sort()).toEqual([ 'classifier_usd', 'episode_count', + 'judge_spend_usd', 'reviewer_direct_fallback_usd', 'reviewer_subagent_usd', 'self_assessment_usd', diff --git a/tools/cost-pricing.mjs b/tools/cost-pricing.mjs index 18822307..825b9e6d 100644 --- a/tools/cost-pricing.mjs +++ b/tools/cost-pricing.mjs @@ -23,3 +23,9 @@ export const PRICING = Object.freeze({ output: 75 / 1_000_000, }), }); + +// Оценочная средняя стоимость одного вызова per-tool LLM-судьи (Sonnet, ~600 in / +// ~10 out токенов на короткий YES/NO-промпт). Используется для derive +// task_cost.judge_spend_usd из числа вызовов судьи (judge_calls). Уточнить, когда +// появится по-вызовный учёт токенов судьи. +export const JUDGE_PER_CALL_USD = 600 * PRICING.sonnet.input + 10 * PRICING.sonnet.output; diff --git a/tools/cost-pricing.test.mjs b/tools/cost-pricing.test.mjs index cedf939f..c8e29f7a 100644 --- a/tools/cost-pricing.test.mjs +++ b/tools/cost-pricing.test.mjs @@ -26,3 +26,11 @@ describe('PRICING constants', () => { expect(Object.isFrozen(PRICING.opus)).toBe(true); }); }); + +describe('JUDGE_PER_CALL_USD', () => { + it('is a positive USD estimate for one per-tool judge call', async () => { + const { JUDGE_PER_CALL_USD } = await import('./cost-pricing.mjs'); + expect(typeof JUDGE_PER_CALL_USD).toBe('number'); + expect(JUDGE_PER_CALL_USD).toBeGreaterThan(0); + }); +}); diff --git a/tools/enforce-hook-helpers.mjs b/tools/enforce-hook-helpers.mjs index 2bcfd117..3b2404fa 100644 --- a/tools/enforce-hook-helpers.mjs +++ b/tools/enforce-hook-helpers.mjs @@ -298,6 +298,52 @@ export function readRationalizationFlags(sessionId) { } catch { return []; } } +export function logJudgeVerdict(sessionId, { tool, verdict }, opts = {}) { + try { + const dir = opts.baseDir || runtimeDir(); + const fi = join(dir, `llm-judge-verdicts-${sessionId || 'unknown'}.jsonl`); + appendFileSync(fi, JSON.stringify({ + ts: new Date().toISOString(), + tool: tool || 'unknown', + verdict: verdict == null ? null : String(verdict), + }) + '\n'); + } catch { /* ignore */ } +} + +export function readJudgeVerdicts(sessionId, opts = {}) { + try { + const dir = opts.baseDir || runtimeDir(); + const fi = join(dir, `llm-judge-verdicts-${sessionId || 'unknown'}.jsonl`); + if (!existsSync(fi)) return []; + return readFileSync(fi, 'utf-8').split('\n').filter(Boolean).map((l) => { + try { return JSON.parse(l); } catch { return null; } + }).filter(Boolean); + } catch { return []; } +} + +export function logSafeBaselineAction(sessionId, { tool, action }, opts = {}) { + try { + const dir = opts.baseDir || runtimeDir(); + const fi = join(dir, `safe-baseline-actions-${sessionId || 'unknown'}.jsonl`); + appendFileSync(fi, JSON.stringify({ + ts: new Date().toISOString(), + tool: tool || 'unknown', + action: String(action || 'allow'), + }) + '\n'); + } catch { /* ignore */ } +} + +export function readSafeBaselineActions(sessionId, opts = {}) { + try { + const dir = opts.baseDir || runtimeDir(); + const fi = join(dir, `safe-baseline-actions-${sessionId || 'unknown'}.jsonl`); + if (!existsSync(fi)) return []; + return readFileSync(fi, 'utf-8').split('\n').filter(Boolean).map((l) => { + try { return JSON.parse(l); } catch { return null; } + }).filter(Boolean); + } catch { return []; } +} + export function readRouterState(sessionId) { try { const p = join(runtimeDir(), `router-state-${sessionId || 'unknown'}.json`); diff --git a/tools/enforce-hook-helpers.test.mjs b/tools/enforce-hook-helpers.test.mjs index 23d33387..21d9a746 100644 --- a/tools/enforce-hook-helpers.test.mjs +++ b/tools/enforce-hook-helpers.test.mjs @@ -23,8 +23,23 @@ import { sessionToolUses, logHookOutcome, runtimeDir, + logSafeBaselineAction, + readSafeBaselineActions, } from './enforce-hook-helpers.mjs'; +describe('safe-baseline action log', () => { + it('appends and reads action records', () => { + const dir = mkdtempSync(join(tmpdir(), 'sb-')); + logSafeBaselineAction('s', { tool: 'Edit', action: 'hard_block' }, { baseDir: dir }); + const recs = readSafeBaselineActions('s', { baseDir: dir }); + expect(recs[0]).toMatchObject({ tool: 'Edit', action: 'hard_block' }); + rmSync(dir, { recursive: true, force: true }); + }); + it('returns [] when no file', () => { + expect(readSafeBaselineActions('nope', { baseDir: tmpdir() })).toEqual([]); + }); +}); + // v4: override surface removed per spec §4.2 — stubs return null/empty describe('v4 override stubs', () => { it('loadOverrideVocab returns empty phrases array (stub)', () => { @@ -520,3 +535,25 @@ describe('detectFullTestRun', () => { expect(detectFullTestRun('cd app && cp src dst')).toBeNull(); }); }); + +// judge verdict log — added for LLM-judge brain-retro factoring +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join as joinPath } from 'node:path'; +import { logJudgeVerdict, readJudgeVerdicts } from './enforce-hook-helpers.mjs'; + +describe('judge verdict log', () => { + it('appends and reads back verdict records (RED: logJudgeVerdict not yet exported)', () => { + const dir = mkdtempSync(joinPath(tmpdir(), 'jv-')); + logJudgeVerdict('sess1', { tool: 'Edit', verdict: 'YES' }, { baseDir: dir }); + logJudgeVerdict('sess1', { tool: 'Bash', verdict: 'NO' }, { baseDir: dir }); + const recs = readJudgeVerdicts('sess1', { baseDir: dir }); + expect(recs.length).toBe(2); + expect(recs[0]).toMatchObject({ tool: 'Edit', verdict: 'YES' }); + expect(typeof recs[0].ts).toBe('string'); + rmSync(dir, { recursive: true, force: true }); + }); + it('returns [] when no file', () => { + expect(readJudgeVerdicts('nope', { baseDir: tmpdir() })).toEqual([]); + }); +}); diff --git a/tools/enforce-llm-judge-per-tool.mjs b/tools/enforce-llm-judge-per-tool.mjs index 7a6438ce..76bbbdf5 100644 --- a/tools/enforce-llm-judge-per-tool.mjs +++ b/tools/enforce-llm-judge-per-tool.mjs @@ -21,7 +21,7 @@ import { judgePerTool, MUTATING_TOOLS, readDeclaredTask, resolveEffectiveTask } from './llm-judge-per-tool.mjs'; import { resolveJudgeConfig } from './llm-judge-config.mjs'; import { readJudgeBudget, bumpJudgeBudget, JUDGE_SESSION_BUDGET, llmJudgeCall } from './llm-judge.mjs'; -import { readStdin, parseEventJson, exitDecision, readTranscript, lastUserPromptText } from './enforce-hook-helpers.mjs'; +import { readStdin, parseEventJson, exitDecision, readTranscript, lastUserPromptText, logJudgeVerdict } from './enforce-hook-helpers.mjs'; import { classifyBashCommand } from './enforce-router-gate.mjs'; /** @@ -117,6 +117,7 @@ export async function runPerTool({ readBudgetImpl, bumpBudgetImpl, llmJudgeCallImpl, + logVerdictImpl = () => {}, sessionBudget = JUDGE_SESSION_BUDGET, }) { // Readonly Bash never mutates → outside the judge's scope; skip (no LLM call, no spend). @@ -146,7 +147,10 @@ export async function runPerTool({ budgetState: { spent, limit: sessionBudget }, llmJudgeCallImpl, }); - if (result.verdict !== undefined) bumpBudgetImpl({ sessionId, by: 1 }); + if (result.verdict !== undefined) { + bumpBudgetImpl({ sessionId, by: 1 }); + logVerdictImpl({ sessionId, tool: event && event.tool_name, verdict: result.block ? 'block' : (result.verdict ?? 'YES') }); + } return result; } @@ -165,6 +169,7 @@ async function main() { readBudgetImpl: readJudgeBudget, bumpBudgetImpl: bumpJudgeBudget, llmJudgeCallImpl: (opts) => llmJudgeCall(opts), + logVerdictImpl: ({ sessionId, tool, verdict }) => logJudgeVerdict(sessionId, { tool, verdict }), }); exitDecision({ block: result.block, message: result.reason }); } catch { diff --git a/tools/enforce-llm-judge-per-tool.test.mjs b/tools/enforce-llm-judge-per-tool.test.mjs index 844b4b5b..bdfefbdf 100644 --- a/tools/enforce-llm-judge-per-tool.test.mjs +++ b/tools/enforce-llm-judge-per-tool.test.mjs @@ -316,6 +316,23 @@ describe('runPerTool — test-runner Bash skips the judge; mutating Bash still j }); }); +describe('runPerTool — verdict logging', () => { + it('runPerTool logs verdict only when a real judge call happened', async () => { + const logged = []; + const r = await runPerTool({ + event: { tool_name: 'Edit', session_id: 's', tool_input: {} }, + judgeConfig: { enabled: true, apiKey: 'k' }, + readDeclaredTaskImpl: () => ({ task_summary: 't' }), + readBudgetImpl: () => 0, + bumpBudgetImpl: () => {}, + llmJudgeCallImpl: async () => 'YES', + logVerdictImpl: (rec) => logged.push(rec), + }); + expect(r.block).toBe(false); + expect(logged).toEqual([{ sessionId: 's', tool: 'Edit', verdict: 'YES' }]); + }); +}); + // Calibration 4 (soft, 2026-05-31): when the classifier summary is "(unknown)", // runPerTool reads the user's last prompt and judges against THAT (better // evidence) instead of an empty task. When the summary is meaningful, the diff --git a/tools/enforce-safe-baseline-metering.mjs b/tools/enforce-safe-baseline-metering.mjs index a22e29cf..de53889e 100644 --- a/tools/enforce-safe-baseline-metering.mjs +++ b/tools/enforce-safe-baseline-metering.mjs @@ -37,6 +37,7 @@ import { lastUserPromptText, lastTurnEntries, exitDecision, + logSafeBaselineAction, } from './enforce-hook-helpers.mjs'; /** @@ -206,6 +207,7 @@ export async function runMain({ event, runtimeDir, transcript: injectedTranscrip const res = runLiveDecision({ event, priorLedger, promptText, currentKeywords, skillMatchedThisTurn }); saveLedger(dir, sess, res.ledger); + if (res.action !== 'allow') logSafeBaselineAction(sess, { tool: event.tool_name, action: res.action }, { baseDir: dir }); if (res.action === 'soft_flag') logFlag(dir, sess, { tool: event.tool_name, reason: res.reason }); if (res.action === 'hard_block') return { block: true, message: `[safe-baseline] ${res.reason}\n${ESCAPE_MSG}` }; return { block: false }; diff --git a/tools/enforce-safe-baseline-metering.test.mjs b/tools/enforce-safe-baseline-metering.test.mjs index c2605d43..989dc867 100644 --- a/tools/enforce-safe-baseline-metering.test.mjs +++ b/tools/enforce-safe-baseline-metering.test.mjs @@ -280,4 +280,19 @@ describe('runMain — live integration', () => { const res = await runMain({ event: { tool_name: 'Skill', session_id: 'S3', transcript_path: tpath }, runtimeDir: dir }); expect(res.block).toBe(false); }); + + it('logs the action to safe-baseline-actions when a mutating tool is hard-blocked', async () => { + const { readFileSync } = await import('node:fs'); + const dir = mkdtempSync(join(tmpdir(), 'sbm-')); + const tpath = join(dir, 't.jsonl'); + writeFileSync(join(dir, 'safe-baseline-ledger-S4.json'), JSON.stringify({ + state: { schema_version: 1, task_id: 't', counts: { Read: 60, Grep: 0, Glob: 0, LS: 0, TodoWrite_writes: 0, AskUserQuestion: 0 }, skill_match_within_task: false }, + lastKeywords: ['router', 'gate', 'safe', 'baseline'], + })); + fixtureTranscript(tpath, [{ type: 'user', message: { role: 'user', content: 'router gate safe baseline' } }]); + const res = await runMain({ event: { tool_name: 'Edit', session_id: 'S4', transcript_path: tpath }, runtimeDir: dir }); + expect(res.block).toBe(true); + const log = readFileSync(join(dir, 'safe-baseline-actions-S4.jsonl'), 'utf-8').trim().split('\n').map((l) => JSON.parse(l)); + expect(log.some((r) => r.action === 'hard_block' && r.tool === 'Edit')).toBe(true); + }); }); diff --git a/tools/observer-transcript-parser.mjs b/tools/observer-transcript-parser.mjs index 5bb155ac..fc37421b 100644 --- a/tools/observer-transcript-parser.mjs +++ b/tools/observer-transcript-parser.mjs @@ -28,6 +28,8 @@ import { buildHookMap, resolveScriptCounts } from './observer-hook-resolver.mjs' // для слепого fallback на heuristic recommended_node — убрано 2026-05-26 // (brain-retro #6 follow-up). Импорты сняты как dead code. import { loadRegistry } from './registry-load.mjs'; +import { extractV4Signals } from './observer-v4-signals.mjs'; +import { JUDGE_PER_CALL_USD } from './cost-pricing.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -847,6 +849,12 @@ export function parseTranscript(transcriptText, fallbackSessionId = null, option const ended_at = stamps[stamps.length - 1] || started_at; const durationMs = new Date(ended_at) - new Date(started_at); + const _v4 = extractV4Signals(sessionId, { + startMs: new Date(started_at).getTime(), + endMs: new Date(ended_at).getTime(), + baseDir: options.runtimeBaseDir || routerStateBaseDir, + }); + const { skills, counts, errors } = collectToolUse(turn); const events = []; @@ -887,7 +895,7 @@ export function parseTranscript(transcriptText, fallbackSessionId = null, option return { schema_version: 4, - schema_minor: 3, + schema_minor: 4, task_id: sessionId, task_ref: sessionId, timestamps: { started_at, ended_at }, @@ -910,7 +918,12 @@ export function parseTranscript(transcriptText, fallbackSessionId = null, option // NB: routerState (line 855) honours routerStateBaseDir option; _state at line 898 // does not (always default dir). Use routerState here so tests with custom temp dir // see the merged values. - task_cost: { ...extractTokenUsage(turn), ...((routerState && routerState.task_cost) || {}) }, + task_cost: { + ...extractTokenUsage(turn), + ...((routerState && routerState.task_cost) || {}), + judge_spend_usd: _v4.judge_calls * JUDGE_PER_CALL_USD, + }, + v4_signals: _v4, // Pass 3 — dynamics meta-block (project-brain-factor-analysis-4passes). // prompt_length_chars: strlen of first user prompt (engagement / clarity proxy). // mcp_servers_used: unique mcp____* fingerprints in this turn. diff --git a/tools/observer-transcript-parser.test.mjs b/tools/observer-transcript-parser.test.mjs index 5edf3251..ffc19cc9 100644 --- a/tools/observer-transcript-parser.test.mjs +++ b/tools/observer-transcript-parser.test.mjs @@ -17,6 +17,21 @@ import { classifyFilePath, } from './observer-transcript-parser.mjs'; +describe('v4_signals + judge_spend_usd in episode', () => { + it('episode carries v4_signals block (default zero/null when no runtime files)', () => { + const ep = parseTranscript(JSON.stringify({ + sessionId: 'no-runtime-sess', + message: { role: 'user', content: 'сделай фичу X' }, + timestamp: '2026-05-31T10:00:00.000Z', + }), null, { routerStateBaseDir: tmpdir() }); + expect(ep.v4_signals).toEqual({ + rationalization_flag_count: 0, judge_verdict: null, safe_baseline_action: null, judge_calls: 0, + }); + expect(typeof ep.task_cost.judge_spend_usd).toBe('number'); + expect(ep.schema_minor).toBe(4); + }); +}); + // Build a JSONL transcript string from entry objects. function jsonl(entries) { return entries.map((e) => JSON.stringify(e)).join('\n'); @@ -236,7 +251,7 @@ describe('parseTranscript', () => { expect(ep.events).toEqual([]); expect(ep.outcome).toBe('unknown'); expect(ep.schema_version).toBe(4); - expect(ep.schema_minor).toBe(3); + expect(ep.schema_minor).toBe(4); }); it('produces a complete 7-field primary_rationale', () => { @@ -469,7 +484,7 @@ describe('parseTranscript — v4 episode (Phase 2 Task 15 bump)', () => { ]); const ep = parseTranscript(t); expect(ep.schema_version).toBe(4); - expect(ep.schema_minor).toBe(3); + expect(ep.schema_minor).toBe(4); expect('classifier_output' in ep).toBe(true); expect('degraded_mode' in ep).toBe(true); expect('classifier_model' in ep.environment).toBe(true); @@ -1047,6 +1062,7 @@ describe('parseTranscript — task_cost integration (Task 2)', () => { self_assessment_input_tokens: 0, self_assessment_output_tokens: 0, reviewer_input_tokens: 0, reviewer_output_tokens: 0, reviewer_subagent_usd: 0, reviewer_direct_fallback_usd: 0, + judge_spend_usd: 0, }); }); }); @@ -1657,7 +1673,7 @@ describe('parseTranscript v3 fields', () => { it('emits schema_version: 4', () => { const ep = parseTranscript(transcriptDirectFeature(), 'sess-1'); expect(ep.schema_version).toBe(4); - expect(ep.schema_minor).toBe(3); + expect(ep.schema_minor).toBe(4); }); it('recommended_node is null on direct episode without router-state (no classifier signal)', () => { @@ -1772,9 +1788,9 @@ describe('parseTranscript — schema v4.3 write-block fields (phase 3 deferred # ].join('\n'); } - it('emits schema_minor 3', () => { + it('emits schema_minor 4', () => { const ep = parseTranscript(simpleTranscript()); - expect(ep.schema_minor).toBe(3); + expect(ep.schema_minor).toBe(4); }); it('emits outcome_reviewed: null', () => { diff --git a/tools/observer-v4-signals.mjs b/tools/observer-v4-signals.mjs new file mode 100644 index 00000000..5d678637 --- /dev/null +++ b/tools/observer-v4-signals.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node +/** + * Pure reader of router-gate v4 runtime signals, scoped to one episode's + * [startMs, endMs] turn window. Feeds observer-transcript-parser (new + * episode.v4_signals block + task_cost.judge_spend_usd). No exec/exit. + * + * Sources (all in ~/.claude/runtime/, per-session): + * - rationalization-flags-.jsonl ({ts, kind, evidence}) + * - llm-judge-verdicts-.jsonl ({ts, tool, verdict}) + * - safe-baseline-actions-.jsonl ({ts, tool, action}) + * - llm-judge-budget-.json ({calls}) + */ +import { readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { homedir } from 'node:os'; + +function rtDir(override) { + return override || join(homedir(), '.claude', 'runtime'); +} + +function readJsonl(path) { + if (!existsSync(path)) return []; + try { + return readFileSync(path, 'utf-8').split('\n').filter(Boolean).map((l) => { + try { return JSON.parse(l); } catch { return null; } + }).filter(Boolean); + } catch { return []; } +} + +function inWindow(rec, startMs, endMs) { + const ms = Date.parse(rec && rec.ts); + return Number.isFinite(ms) && ms >= startMs && ms <= endMs; +} + +// Severity order: hard_block > soft_flag > allow. +const ACTION_RANK = { allow: 0, soft_flag: 1, hard_block: 2 }; +function worstAction(records) { + let best = null; + for (const r of records) { + const a = r && r.action; + if (a == null) continue; + if (best === null || (ACTION_RANK[a] ?? -1) > (ACTION_RANK[best] ?? -1)) best = a; + } + return best; +} + +export function extractV4Signals(sessionId, { startMs, endMs, baseDir } = {}) { + const dir = rtDir(baseDir); + const sess = sessionId || 'unknown'; + + const flags = readJsonl(join(dir, `rationalization-flags-${sess}.jsonl`)) + .filter((r) => inWindow(r, startMs, endMs)); + + const verdicts = readJsonl(join(dir, `llm-judge-verdicts-${sess}.jsonl`)) + .filter((r) => inWindow(r, startMs, endMs)); + const judge_verdict = verdicts.length ? (verdicts[verdicts.length - 1].verdict ?? null) : null; + + const actions = readJsonl(join(dir, `safe-baseline-actions-${sess}.jsonl`)) + .filter((r) => inWindow(r, startMs, endMs)); + const safe_baseline_action = worstAction(actions); + + let judge_calls = 0; + const bp = join(dir, `llm-judge-budget-${sess}.json`); + if (existsSync(bp)) { + try { judge_calls = Number(JSON.parse(readFileSync(bp, 'utf-8')).calls) || 0; } catch { /* keep 0 */ } + } + + return { + rationalization_flag_count: flags.length, + judge_verdict, + safe_baseline_action, + judge_calls, + }; +} diff --git a/tools/observer-v4-signals.test.mjs b/tools/observer-v4-signals.test.mjs new file mode 100644 index 00000000..75f3f9e8 --- /dev/null +++ b/tools/observer-v4-signals.test.mjs @@ -0,0 +1,49 @@ +import { describe, it, expect } from 'vitest'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { extractV4Signals } from './observer-v4-signals.mjs'; + +function write(dir, name, lines) { + writeFileSync(join(dir, name), lines.map((l) => JSON.stringify(l)).join('\n') + '\n'); +} + +describe('extractV4Signals', () => { + it('counts rationalization flags inside the turn window only', () => { + const dir = mkdtempSync(join(tmpdir(), 'v4-')); + write(dir, 'rationalization-flags-s.jsonl', [ + { ts: '2026-05-31T10:00:00.000Z', kind: 'x' }, // before window + { ts: '2026-05-31T10:05:00.000Z', kind: 'y' }, // in + { ts: '2026-05-31T10:06:00.000Z', kind: 'z' }, // in + ]); + write(dir, 'llm-judge-verdicts-s.jsonl', [ + { ts: '2026-05-31T10:05:30.000Z', tool: 'Edit', verdict: 'YES' }, + { ts: '2026-05-31T10:05:40.000Z', tool: 'Bash', verdict: 'block' }, + ]); + write(dir, 'safe-baseline-actions-s.jsonl', [ + { ts: '2026-05-31T10:05:10.000Z', tool: 'Edit', action: 'soft_flag' }, + { ts: '2026-05-31T10:05:50.000Z', tool: 'Write', action: 'hard_block' }, + ]); + writeFileSync(join(dir, 'llm-judge-budget-s.json'), JSON.stringify({ calls: 7 })); + const sig = extractV4Signals('s', { + startMs: Date.parse('2026-05-31T10:04:00.000Z'), + endMs: Date.parse('2026-05-31T10:07:00.000Z'), + baseDir: dir, + }); + expect(sig.rationalization_flag_count).toBe(2); + expect(sig.judge_verdict).toBe('block'); // last in-window verdict + expect(sig.safe_baseline_action).toBe('hard_block'); // worst in-window action + expect(sig.judge_calls).toBe(7); + rmSync(dir, { recursive: true, force: true }); + }); + + it('returns zero/null defaults when files absent', () => { + const sig = extractV4Signals('nope', { startMs: 0, endMs: 1, baseDir: tmpdir() }); + expect(sig).toEqual({ + rationalization_flag_count: 0, + judge_verdict: null, + safe_baseline_action: null, + judge_calls: 0, + }); + }); +});