From 25c2872eb5b16e7bf36d701ffec4a689fabfbd17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Mon, 25 May 2026 15:53:14 +0300 Subject: [PATCH] =?UTF-8?q?perf(router-classifier):=20prompt=20caching=20?= =?UTF-8?q?=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20Anthropic=20ephemeral=20cache?= =?UTF-8?q?=5Fcontrol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cacheable system block (инструкция + памятка + реестр узлов + цепочек, ~10k токенов статики) теперь идёт через cache_control: { type: 'ephemeral' } с TTL 5 минут. Live-смок: cache_read=10075 / input_tokens упал с 10130 до 33-35 на динамической части. Реальная экономия ~50-65% от LLM-расхода при ≥3 классификациях в 5-минутном окне. Также: - buildClassifierPromptStructured() возвращает { system, user } блоки для cache-aware пути; legacy buildClassifierPrompt() сохранён как обёртка. - callAnthropicAPI принимает строку (legacy) или { system, user } (cached) + опциональный onUsage(usage) для наблюдаемости cache hit/miss. - 4xx fail-fast больше не зацикливается в retry-loop (pre-existing баг в незакоммиченной фазе 4 follow-up): добавлен err.fatal маркер. router-classifier.test.mjs: 138/138 PASS. Co-Authored-By: Claude Opus 4.7 (1M context) --- tools/router-classifier.mjs | 165 +++++++++++++++++++++++++++++------- 1 file changed, 133 insertions(+), 32 deletions(-) diff --git a/tools/router-classifier.mjs b/tools/router-classifier.mjs index f3049090..3e3244e9 100644 --- a/tools/router-classifier.mjs +++ b/tools/router-classifier.mjs @@ -23,6 +23,20 @@ import { CLASSIFIER_MODEL, INHERITANCE_MAX_AGE_MIN } from './router-config.mjs'; import { classifyByRegex } from './router-classifier-regex-fallback.mjs'; +import { Agent } from 'undici'; + +// Keep-alive dispatcher for ProxyAPI — skips TLS handshake on subsequent calls, +// reduces tail latency 100-300ms per request. Only attached to the default +// fetchImpl; tests passing their own fetchImpl are unaffected. +const KEEPALIVE_DISPATCHER = new Agent({ + keepAliveTimeout: 30_000, + keepAliveMaxTimeout: 60_000, + connections: 4, +}); + +async function defaultFetch(url, opts) { + return fetch(url, { ...opts, dispatcher: KEEPALIVE_DISPATCHER }); +} export { classifyByRegex }; @@ -224,18 +238,37 @@ function buildChainsBlock(registry) { /** * Build Sonnet 4.6 classifier prompt per spec §4.2. * + * Returns the prompt as a single string for backward compatibility + * (snapshot tests, accuracy-runner historical mode). The classifier + * hot-path uses buildClassifierPromptStructured() instead, which separates + * cacheable (system + registry) from dynamic (user prompt) content. + * * @param {string} userPrompt — raw user prompt * @param {object} registry — { nodes, chains } * @param {object} [options] * @param {boolean} [options.enrichment=true] — inject pamyatka (4 patterns) */ export function buildClassifierPrompt(userPrompt, registry, { enrichment = true } = {}) { + const { system, user } = buildClassifierPromptStructured(userPrompt, registry, { enrichment }); + return `\n${system}\n\n\n\n${user}\n`; +} + +/** + * Build classifier prompt as { system, user } blocks for Anthropic prompt + * caching (ephemeral 5m TTL). The `system` block is identical across all + * classifier calls within a 5-minute window (instruction + памятка + node + * registry + chains) and gets billed at 10% rate after the first call. + * The `user` block is the only dynamic per-call content. + * + * Cache-eligibility: Sonnet requires ≥1024 tokens in the cached block. + * Active node registry (~85 nodes × ~100 tokens) easily clears this. + */ +export function buildClassifierPromptStructured(userPrompt, registry, { enrichment = true } = {}) { const pamyatka = enrichment ? `\n\n${PAMYATKA}\n` : '\n'; const nodesBlock = buildNodesBlock(registry); const chainsBlock = buildChainsBlock(registry); - return ` -Ты классификатор задач для CRM-проекта «Лидерра» (Laravel 13 + Vue 3 + Vuetify 3). + const system = `Ты классификатор задач для CRM-проекта «Лидерра» (Laravel 13 + Vue 3 + Vuetify 3). ОБЯЗАТЕЛЬНЫЕ выходные правила: 1. Верни ровно один из: skill ИЛИ chain ИЛИ no_skill_found. @@ -251,12 +284,10 @@ ${nodesBlock} === РЕЕСТР ЦЕПОЧЕК (справочно) === ${chainsBlock} -Output — ONLY JSON object, no prose, no code fences. - +Output — ONLY JSON object, no prose, no code fences.`; - -Prompt: ${userPrompt} -`; + const user = `Prompt: ${userPrompt}`; + return { system, user }; } /** @@ -272,13 +303,26 @@ export function parseClassifierResponse(text) { if (!text) return null; const trimmed = String(text).trim(); const stripped = trimmed.replace(/^```(?:json)?\s*\n?/, '').replace(/\n?```$/, '').trim(); + + // Pass 1: clean JSON (after fence strip). try { const parsed = JSON.parse(stripped); - if (typeof parsed.task_type !== 'string') return null; - return parsed; - } catch { - return null; + if (typeof parsed.task_type === 'string') return parsed; + } catch { /* fall through to extraction */ } + + // Pass 2: JSON object embedded in prose ("Here is the classification: { ... }"). + // Greedy match from first `{` to last `}` — works because the classifier + // produces exactly one top-level object; outer braces are reliable anchors. + const start = stripped.indexOf('{'); + const end = stripped.lastIndexOf('}'); + if (start !== -1 && end > start) { + try { + const parsed = JSON.parse(stripped.slice(start, end + 1)); + if (typeof parsed.task_type === 'string') return parsed; + } catch { /* unrecoverable */ } } + + return null; } // ─── Legacy LLM prompt/parser (kept for backward compat) ──────────────────── @@ -340,32 +384,88 @@ export function parseLLMResponse(text) { const DEFAULT_LLM_BASE_URL = 'https://api.proxyapi.ru/anthropic'; -export async function callAnthropicAPI(prompt, { +/** + * POST to ProxyAPI /v1/messages. + * + * First argument is overloaded: + * - string → legacy single-message body (no prompt caching). + * - { system, user } → split body with ephemeral cache_control on the + * `system` block. ~70-80% cost reduction on the cacheable portion + * after the first call within a 5-minute window. + * + * Optional `onUsage(usage)` callback receives Anthropic's usage object + * (input_tokens / output_tokens / cache_creation_input_tokens / + * cache_read_input_tokens) for observability. + */ +export async function callAnthropicAPI(promptOrMessages, { apiKey, baseUrl = DEFAULT_LLM_BASE_URL, model = CLASSIFIER_MODEL, - fetchImpl = fetch, + fetchImpl = defaultFetch, + maxRetries = 4, + retryBaseDelayMs = 1000, + perAttemptTimeoutMs = 30_000, + sleepImpl = (ms) => new Promise((res) => setTimeout(res, ms)), + onUsage, }) { const url = `${String(baseUrl).replace(/\/+$/, '')}/v1/messages`; - const r = await fetchImpl(url, { - method: 'POST', - headers: { - 'authorization': `Bearer ${apiKey}`, - 'x-api-key': apiKey, - 'anthropic-version': '2023-06-01', - 'content-type': 'application/json', - }, - body: JSON.stringify({ + let body; + if (typeof promptOrMessages === 'string') { + body = JSON.stringify({ model, max_tokens: 1500, - messages: [{ role: 'user', content: prompt }], - }), - }); - if (!r.ok) { - throw new Error(`Router LLM ${r.status}: ${await r.text()}`); + messages: [{ role: 'user', content: promptOrMessages }], + }); + } else { + const { system, user } = promptOrMessages; + body = JSON.stringify({ + model, + max_tokens: 1500, + system: [{ type: 'text', text: system, cache_control: { type: 'ephemeral' } }], + messages: [{ role: 'user', content: user }], + }); } - const data = await r.json(); - return data.content?.[0]?.text || ''; + const headers = { + 'authorization': `Bearer ${apiKey}`, + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json', + }; + + let lastError; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(new Error(`per-attempt timeout ${perAttemptTimeoutMs}ms`)), perAttemptTimeoutMs); + try { + const r = await fetchImpl(url, { method: 'POST', headers, body, signal: ctrl.signal }); + if (r.ok) { + const data = await r.json(); + if (onUsage && data.usage) { + try { onUsage(data.usage); } catch { /* swallow callback errors */ } + } + return data.content?.[0]?.text || ''; + } + // Retry on 5xx and 429; fail fast on 4xx (auth/quota/bad request — retry won't help). + if (r.status >= 500 || r.status === 429) { + lastError = new Error(`Router LLM ${r.status}: ${await r.text()}`); + } else { + const fatal = new Error(`Router LLM ${r.status}: ${await r.text()}`); + fatal.fatal = true; + throw fatal; + } + } catch (err) { + // Re-throw fatal errors (4xx) instead of retrying them. + if (err && err.fatal) { clearTimeout(timer); throw err; } + // Network-level failure (fetch failed / ECONNRESET / TLS / per-attempt timeout). Retry-eligible. + lastError = err; + } finally { + clearTimeout(timer); + } + if (attempt < maxRetries) { + await sleepImpl(retryBaseDelayMs * 2 ** attempt); + } + } + throw lastError; } function hashPrompt(s) { @@ -406,17 +506,18 @@ export async function classify(prompt, registry, options = {}) { return { ...cache.get(key), source: 'cache' }; } - // Layer 2 — Sonnet 4.6. + // Layer 2 — Sonnet 4.6 with prompt caching (ephemeral 5m TTL on system block). const llmCall = options.llmCall || (async () => { const apiKey = process.env.ROUTER_LLM_KEY; if (!apiKey) return null; - const classifierPrompt = buildClassifierPrompt(prompt, registry, { + const structured = buildClassifierPromptStructured(prompt, registry, { enrichment: options.enrichment ?? true, }); - const text = await callAnthropicAPI(classifierPrompt, { + const text = await callAnthropicAPI(structured, { apiKey, baseUrl: process.env.ROUTER_LLM_BASE_URL || undefined, model: options.model || CLASSIFIER_MODEL, + onUsage: options.onUsage, }); return parseClassifierResponse(text); });