From 2798cfcd50c62558dfec7a5d37eb41968148f5cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Thu, 28 May 2026 07:48:20 +0300 Subject: [PATCH] perf(brain-retro): prompt-caching split on reviewer-agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add buildReviewPromptStructured() returning { system, user } and route reviewViaDirectApi through callAnthropicAPI's structured branch — same pattern the classifier already uses (router-classifier.mjs L456-484), so infrastructure is reused, no new transport code. system block: static instructions + 8-dim cues + schema-version notes (byte-identical across episodes of the same schema_version → cache key stable within a 5-min TTL). user block: per-episode JSON (volatile). Effect on Opus 4.7: ~zero until system grows past 4096-token cache- minimum or model switches to Sonnet (2048 min). Anthropic silently no-ops cache_control when prefix is below the minimum — no error, cache_creation_input_tokens just stays at 0. Architecturally correct and future-proof; activates the moment either condition flips. buildReviewPrompt() kept as backward-compat wrapper. Tests: +5 invariants for the split + cache-prerequisite check (system identical across two v4 episodes with different bodies). 14/14 GREEN. ремонт: фикс инфраструктуры стоимости — split prompt для активации prompt caching на reviewer-agent Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/observer/STATUS.md | 28 ++++----- tools/brain-retro-opus-reviewer.mjs | 72 ++++++++++++++++-------- tools/brain-retro-opus-reviewer.test.mjs | 41 +++++++++++++- 3 files changed, 103 insertions(+), 38 deletions(-) diff --git a/docs/observer/STATUS.md b/docs/observer/STATUS.md index 8e3e074a..c440581f 100644 --- a/docs/observer/STATUS.md +++ b/docs/observer/STATUS.md @@ -1,6 +1,6 @@ # Brain Status (auto-generated) -Last updated: 2026-05-28T02:37:38.704Z +Last updated: 2026-05-28T03:47:41.838Z | Контролёр | Состояние | Детали | |---|---|---| @@ -8,13 +8,13 @@ Last updated: 2026-05-28T02:37:38.704Z | C2 Cross-ref consistency | ✅ | [cross-ref-checker] OK — 0 drift in 4 files | | C3 Observer-of-observer | ✅ | [observer-of-observer] OK — last read 0 week(s) ago | | C4 Сигнальный статус | ✅ | This file (self-reference) | -| C5 Observer-coverage | ⚠️ | 715 episode(s) this month · Stop-hook + post-commit OK · 20 missed activation(s) — see /brain-retro | +| C5 Observer-coverage | ⚠️ | 726 episode(s) this month · Stop-hook + post-commit OK · 20 missed activation(s) — see /brain-retro | | C6 Chain map sync | ✅ | [chain-map-checker] OK — 16 chains in sync | ## Метрики (информационные, не алерты) -- Observer evidence: 715 episodes this month, 0 observer_error markers, 158 PII matches before filter -- Legacy v1 episodes (not in factor analysis): 576 +- Observer evidence: 726 episodes this month, 0 observer_error markers, 160 PII matches before filter +- Legacy v1 episodes (not in factor analysis): 587 - Last /brain-retro: 0 day(s) ago - Использование узлов: см. `/brain-retro` (раз в спринт). missed_activations: 20. **Неиспользованные узлы — не алерт, если профильной задачи не было** (Pravila §16.4 v1.36; capability-readiness; см. memory `feedback_brain_unused_tools_not_problem` — outside-repo memory store). @@ -24,16 +24,16 @@ Baseline дисциплины роутера (этап 2 router discipline overh | Тип задачи | Эпизодов | % с триггер-матчем | % через скил | |---|---|---|---| -| analysis | 28 | 32.1% | 14.3% | +| analysis | 29 | 31.0% | 13.8% | | bugfix | 20 | 20.0% | 25.0% | -| planning | 17 | 17.6% | 17.6% | -| feature | 16 | 12.5% | 0.0% | +| planning | 18 | 16.7% | 16.7% | +| feature | 17 | 11.8% | 0.0% | | cleanup | 7 | 0.0% | 0.0% | | refactor | 1 | 0.0% | 0.0% | -Router step distribution: 1: 303, 2: 265, 3: 75, 5: 64 +Router step distribution: 1: 308, 2: 269, 3: 76, 5: 65 -Boundaries applied (ADR / границы): 88 of 707 эпизодов (12.4%). +Boundaries applied (ADR / границы): 88 of 718 эпизодов (12.3%). ## Активные многоэтапные проекты @@ -51,10 +51,10 @@ Boundaries applied (ADR / границы): 88 of 707 эпизодов (12.4%). | Компонент | Токены (in/out) | USD | |---|---|---| -| Classifier (Sonnet 4.6) | 7217/72657 | $1.11 | +| Classifier (Sonnet 4.6) | 7990/77078 | $1.18 | | Self-assessment (Sonnet 4.6) | 0/0 | $0.00 | | Reviewer (Opus 4.7 + fallback) | 0/0 | $0.00 | -| **Итого** | | **$1.11** | +| **Итого** | | **$1.18** | ## Аномалии классификатора @@ -67,7 +67,7 @@ Episodes since last run: 609 / threshold: 10 ## Reviewer: субагент vs fallback -0 эпизодов проверено из 715. +0 эпизодов проверено из 726. ## Reviewer findings @@ -109,10 +109,10 @@ Episodes since last run: 609 / threshold: 10 | Фраза | За всё время | За сегодня | |---|---|---| -| `recovery` | 286 | 13 ⚠️ | +| `recovery` | 311 | 38 ⚠️ | | `ремонт инфраструктуры` | 185 | 26 ⚠️ | | `срочно` | 88 | 6 ⚠️ | -| `без скилов` | 60 | 2 | +| `без скилов` | 75 | 17 ⚠️ | | `memory dump` | 8 | 0 | | `direct ok` | 6 | 0 | | `быстрый коммит` | 3 | 0 | diff --git a/tools/brain-retro-opus-reviewer.mjs b/tools/brain-retro-opus-reviewer.mjs index 5ad8a778..8ea4823c 100644 --- a/tools/brain-retro-opus-reviewer.mjs +++ b/tools/brain-retro-opus-reviewer.mjs @@ -28,27 +28,32 @@ const REQUIRED_REVIEW_FIELDS = [ ]; /** - * Build the adaptive review prompt for a given episode. Pure. + * Build the adaptive review prompt as { system, user } blocks for Anthropic + * prompt caching (ephemeral 5m TTL). The `system` block contains static + * instructions + 8-dim cues + schema-version-conditional notes; it is + * byte-identical across all episodes with the same schema_version and gets + * billed at ~10% rate after the first call within a 5-minute window. The + * `user` block carries the per-episode JSON (volatile). * - * Adaptive prompt template (spec §4.6): - * - v4 → full prompt including alternatives_considered, self_assessment, - * chain_gaps cues. - * - v3 → omits alternatives_considered. - * - v2 → omits both alternatives_considered and self_assessment. - * - v1 → skipped upstream (caller filters them out). + * Cache-eligibility caveat: Anthropic's minimum cacheable prefix is + * model-dependent — 4096 tokens on Opus 4.7 / 4.6, 2048 on Sonnet 4.6. The + * static block here is ~300-400 tokens, so on Opus 4.7 cache writes silently + * no-op until either the static block grows or the model switches to Sonnet. + * The split is still applied so the moment either condition flips, caching + * activates with zero further code changes. */ -export function buildReviewPrompt(episode) { +export function buildReviewPromptStructured(episode) { const v = Number(episode?.schema_version) || 0; - const cues = []; - - cues.push('node_quality: correct | wrong_node | overkill | underkill | disputable'); - cues.push('chain_quality: correct | missing_step | extra_step | wrong_order | n/a'); - cues.push('gap_assessment: acceptable | mistake_should_complete | mistake_should_not_start | n/a'); - cues.push('agent_self_assessment_accuracy: accurate | over_confident | under_confident | no_self_assessment'); - cues.push('error_root_cause: wrong_skill | wrong_tool | wrong_chain_order | external_failure | n/a'); - cues.push('alternative_better: | null'); - cues.push('outcome_reviewed: success | soft_success | rework | blocked'); - cues.push('reasoning: 1-3 sentences'); + const cues = [ + 'node_quality: correct | wrong_node | overkill | underkill | disputable', + 'chain_quality: correct | missing_step | extra_step | wrong_order | n/a', + 'gap_assessment: acceptable | mistake_should_complete | mistake_should_not_start | n/a', + 'agent_self_assessment_accuracy: accurate | over_confident | under_confident | no_self_assessment', + 'error_root_cause: wrong_skill | wrong_tool | wrong_chain_order | external_failure | n/a', + 'alternative_better: | null', + 'outcome_reviewed: success | soft_success | rework | blocked', + 'reasoning: 1-3 sentences', + ]; const adaptiveNotes = []; if (v >= 3) { @@ -60,7 +65,7 @@ export function buildReviewPrompt(episode) { adaptiveNotes.push('execution_trace.chain_gaps shows whether the recommended chain ran in full.'); } - return [ + const system = [ 'You are the independent reviewer of routing decisions for the Лидерра brain-governance experiment.', 'Return ONLY a JSON object with the 8 fields below. No prose, no code fences.', '', @@ -69,12 +74,32 @@ export function buildReviewPrompt(episode) { '', adaptiveNotes.length ? 'Notes for this schema version:' : '', ...adaptiveNotes.map((n) => ' - ' + n), - '', + ].filter(Boolean).join('\n'); + + const user = [ 'Episode (JSON):', JSON.stringify(episode, null, 2), '', 'Output JSON only.', - ].filter(Boolean).join('\n'); + ].join('\n'); + + return { system, user }; +} + +/** + * Backward-compat wrapper — returns the concatenated single-string prompt for + * tests and any caller that hasn't switched to the structured form. + * + * Adaptive prompt template (spec §4.6): + * - v4 → full prompt including alternatives_considered, self_assessment, + * chain_gaps cues. + * - v3 → omits alternatives_considered. + * - v2 → omits both alternatives_considered and self_assessment. + * - v1 → skipped upstream (caller filters them out). + */ +export function buildReviewPrompt(episode) { + const { system, user } = buildReviewPromptStructured(episode); + return `${system}\n\n${user}`; } /** @@ -113,12 +138,13 @@ export async function reviewViaDirectApi(episode, options = {}) { const { callAnthropicAPI } = await import('./router-classifier.mjs'); const apiKey = options.apiKey ?? process.env.ROUTER_LLM_KEY; if (!apiKey) return null; - const prompt = buildReviewPrompt(episode); + const structured = buildReviewPromptStructured(episode); try { - const text = await callAnthropicAPI(prompt, { + const text = await callAnthropicAPI(structured, { apiKey, baseUrl: options.baseUrl ?? process.env.ROUTER_LLM_BASE_URL ?? undefined, model: options.model ?? REVIEWER_MODEL, + onUsage: options.onUsage, }); return parseReview(text); } catch { diff --git a/tools/brain-retro-opus-reviewer.test.mjs b/tools/brain-retro-opus-reviewer.test.mjs index 28b03232..00402eef 100644 --- a/tools/brain-retro-opus-reviewer.test.mjs +++ b/tools/brain-retro-opus-reviewer.test.mjs @@ -1,6 +1,6 @@ // tools/brain-retro-opus-reviewer.test.mjs — TDD for Phase 3 Task 18 (G16, spec §4.6) import { describe, it, expect } from 'vitest'; -import { buildReviewPrompt, parseReview } from './brain-retro-opus-reviewer.mjs'; +import { buildReviewPrompt, buildReviewPromptStructured, parseReview } from './brain-retro-opus-reviewer.mjs'; describe('buildReviewPrompt — adaptive v2/v3/v4 (spec §4.6)', () => { it('v4 includes alternatives_considered + self_assessment + chain_gaps cues', () => { @@ -40,6 +40,45 @@ describe('buildReviewPrompt — adaptive v2/v3/v4 (spec §4.6)', () => { }); }); +describe('buildReviewPromptStructured — split for prompt caching', () => { + it('returns { system, user } with episode JSON only in user block', () => { + const ep = { schema_version: 4, task_id: 'task-cache-1' }; + const { system, user } = buildReviewPromptStructured(ep); + expect(typeof system).toBe('string'); + expect(typeof user).toBe('string'); + expect(user).toContain('task-cache-1'); + expect(system).not.toContain('task-cache-1'); + }); + + it('system block carries the 8-dim cue contract', () => { + const { system } = buildReviewPromptStructured({ schema_version: 4 }); + expect(system).toContain('node_quality'); + expect(system).toContain('chain_quality'); + expect(system).toContain('outcome_reviewed'); + expect(system).toContain('reasoning'); + }); + + it('system block is identical across two v4 episodes with different bodies (cache prerequisite)', () => { + const a = buildReviewPromptStructured({ schema_version: 4, task_id: 'a' }).system; + const b = buildReviewPromptStructured({ schema_version: 4, task_id: 'b', other: 'noise' }).system; + expect(a).toBe(b); + }); + + it('user block carries Episode (JSON): marker + final "Output JSON only."', () => { + const { user } = buildReviewPromptStructured({ schema_version: 4, task_id: 't' }); + expect(user).toContain('Episode (JSON):'); + expect(user.trim().endsWith('Output JSON only.')).toBe(true); + }); + + it('buildReviewPrompt remains a string concatenation of system + user (backward compat)', () => { + const ep = { schema_version: 4, task_id: 'compat-1' }; + const { system, user } = buildReviewPromptStructured(ep); + const combined = buildReviewPrompt(ep); + expect(combined).toContain(system); + expect(combined).toContain(user); + }); +}); + describe('parseReview — 8-dim review schema (spec §4.6)', () => { it('parses a complete 8-dim review JSON', () => { const r = parseReview('{"node_quality":"correct","chain_quality":"n/a","gap_assessment":"n/a","agent_self_assessment_accuracy":"accurate","error_root_cause":"n/a","alternative_better":null,"outcome_reviewed":"success","reasoning":"x"}');