liderra/portal
1
0
Fork 0
Code Issues Pull Requests Actions 73 Packages Projects Releases Wiki Activity

perf(router-classifier): prompt caching через Anthropic ephemeral cache_control

Browse Source 
Cacheable system block (инструкция + памятка + реестр узлов + цепочек,
~10k токенов статики) теперь идёт через cache_control: { type: 'ephemeral' }
с TTL 5 минут. Live-смок: cache_read=10075 / input_tokens упал с 10130 до 33-35
на динамической части. Реальная экономия ~50-65% от LLM-расхода при
≥3 классификациях в 5-минутном окне.

Также:
- buildClassifierPromptStructured() возвращает { system, user } блоки для
  cache-aware пути; legacy buildClassifierPrompt() сохранён как обёртка.
- callAnthropicAPI принимает строку (legacy) или { system, user } (cached)
  + опциональный onUsage(usage) для наблюдаемости cache hit/miss.
- 4xx fail-fast больше не зацикливается в retry-loop (pre-existing баг
  в незакоммиченной фазе 4 follow-up): добавлен err.fatal маркер.

router-classifier.test.mjs: 138/138 PASS.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Дмитрий
2026-05-25 15:53:14 +03:00
parent de333852e7
commit 25c2872eb5
1 changed files with 133 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/router-classifier.mjs
+133 -32
View File
@@ -23,6 +23,20 @@
import { CLASSIFIER_MODEL, INHERITANCE_MAX_AGE_MIN } from './router-config.mjs';
import { classifyByRegex } from './router-classifier-regex-fallback.mjs';
import { Agent } from 'undici';
// Keep-alive dispatcher for ProxyAPI — skips TLS handshake on subsequent calls,
// reduces tail latency 100-300ms per request. Only attached to the default
// fetchImpl; tests passing their own fetchImpl are unaffected.
const KEEPALIVE_DISPATCHER = new Agent({
keepAliveTimeout: 30_000,
keepAliveMaxTimeout: 60_000,
connections: 4,
});
async function defaultFetch(url, opts) {
return fetch(url, { ...opts, dispatcher: KEEPALIVE_DISPATCHER });
}
export { classifyByRegex };
@@ -224,18 +238,37 @@ function buildChainsBlock(registry) {
/**
* Build Sonnet 4.6 classifier prompt per spec §4.2.
*
* Returns the prompt as a single string for backward compatibility
* (snapshot tests, accuracy-runner historical mode). The classifier
* hot-path uses buildClassifierPromptStructured() instead, which separates
* cacheable (system + registry) from dynamic (user prompt) content.
*
* @param {string} userPrompt — raw user prompt
* @param {object} registry — { nodes, chains }
* @param {object} [options]
* @param {boolean} [options.enrichment=true] — inject pamyatka (4 patterns)
*/
export function buildClassifierPrompt(userPrompt, registry, { enrichment = true } = {}) {
const { system, user } = buildClassifierPromptStructured(userPrompt, registry, { enrichment });
return `<system>\n${system}\n</system>\n\n<user>\n${user}\n</user>`;
}
/**
* Build classifier prompt as { system, user } blocks for Anthropic prompt
* caching (ephemeral 5m TTL). The `system` block is identical across all
* classifier calls within a 5-minute window (instruction + памятка + node
* registry + chains) and gets billed at 10% rate after the first call.
* The `user` block is the only dynamic per-call content.
*
* Cache-eligibility: Sonnet requires ≥1024 tokens in the cached block.
* Active node registry (~85 nodes × ~100 tokens) easily clears this.
*/
export function buildClassifierPromptStructured(userPrompt, registry, { enrichment = true } = {}) {
const pamyatka = enrichment ? `\n\n${PAMYATKA}\n` : '\n';
const nodesBlock = buildNodesBlock(registry);
const chainsBlock = buildChainsBlock(registry);
return `<system>
Ты классификатор задач для CRM-проекта «Лидерра» (Laravel 13 + Vue 3 + Vuetify 3).
const system = `Ты классификатор задач для CRM-проекта «Лидерра» (Laravel 13 + Vue 3 + Vuetify 3).
ОБЯЗАТЕЛЬНЫЕ выходные правила:
1. Верни ровно один из: skill ИЛИ chain ИЛИ no_skill_found.
@@ -251,12 +284,10 @@ ${nodesBlock}
=== РЕЕСТР ЦЕПОЧЕК (справочно) ===
${chainsBlock}
Output — ONLY JSON object, no prose, no code fences.
</system>
Output — ONLY JSON object, no prose, no code fences.`;
<user>
Prompt: ${userPrompt}
</user>`;
const user = `Prompt: ${userPrompt}`;
return { system, user };
}
/**
@@ -272,13 +303,26 @@ export function parseClassifierResponse(text) {
if (!text) return null;
const trimmed = String(text).trim();
const stripped = trimmed.replace(/^```(?:json)?\s*\n?/, '').replace(/\n?```$/, '').trim();
// Pass 1: clean JSON (after fence strip).
try {
const parsed = JSON.parse(stripped);
if (typeof parsed.task_type !== 'string') return null;
return parsed;
} catch {
return null;
if (typeof parsed.task_type === 'string') return parsed;
} catch { /* fall through to extraction */ }
// Pass 2: JSON object embedded in prose ("Here is the classification: { ... }").
// Greedy match from first `{` to last `}` — works because the classifier
// produces exactly one top-level object; outer braces are reliable anchors.
const start = stripped.indexOf('{');
const end = stripped.lastIndexOf('}');
if (start !== -1 && end > start) {
try {
const parsed = JSON.parse(stripped.slice(start, end + 1));
if (typeof parsed.task_type === 'string') return parsed;
} catch { /* unrecoverable */ }
}
return null;
}
// ─── Legacy LLM prompt/parser (kept for backward compat) ────────────────────
@@ -340,32 +384,88 @@ export function parseLLMResponse(text) {
const DEFAULT_LLM_BASE_URL = 'https://api.proxyapi.ru/anthropic';
export async function callAnthropicAPI(prompt, {
/**
* POST to ProxyAPI /v1/messages.
*
* First argument is overloaded:
* - string → legacy single-message body (no prompt caching).
* - { system, user } → split body with ephemeral cache_control on the
* `system` block. ~70-80% cost reduction on the cacheable portion
* after the first call within a 5-minute window.
*
* Optional `onUsage(usage)` callback receives Anthropic's usage object
* (input_tokens / output_tokens / cache_creation_input_tokens /
* cache_read_input_tokens) for observability.
*/
export async function callAnthropicAPI(promptOrMessages, {
apiKey,
baseUrl = DEFAULT_LLM_BASE_URL,
model = CLASSIFIER_MODEL,
fetchImpl = fetch,
fetchImpl = defaultFetch,
maxRetries = 4,
retryBaseDelayMs = 1000,
perAttemptTimeoutMs = 30_000,
sleepImpl = (ms) => new Promise((res) => setTimeout(res, ms)),
onUsage,
}) {
const url = `${String(baseUrl).replace(/\/+$/, '')}/v1/messages`;
const r = await fetchImpl(url, {
method: 'POST',
headers: {
'authorization': `Bearer ${apiKey}`,
'x-api-key': apiKey,
'anthropic-version': '2023-06-01',
'content-type': 'application/json',
},
body: JSON.stringify({
let body;
if (typeof promptOrMessages === 'string') {
body = JSON.stringify({
model,
max_tokens: 1500,
messages: [{ role: 'user', content: prompt }],
}),
});
if (!r.ok) {
throw new Error(`Router LLM ${r.status}: ${await r.text()}`);
messages: [{ role: 'user', content: promptOrMessages }],
});
} else {
const { system, user } = promptOrMessages;
body = JSON.stringify({
model,
max_tokens: 1500,
system: [{ type: 'text', text: system, cache_control: { type: 'ephemeral' } }],
messages: [{ role: 'user', content: user }],
});
}
const data = await r.json();
return data.content?.[0]?.text || '';
const headers = {
'authorization': `Bearer ${apiKey}`,
'x-api-key': apiKey,
'anthropic-version': '2023-06-01',
'content-type': 'application/json',
};
let lastError;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(new Error(`per-attempt timeout ${perAttemptTimeoutMs}ms`)), perAttemptTimeoutMs);
try {
const r = await fetchImpl(url, { method: 'POST', headers, body, signal: ctrl.signal });
if (r.ok) {
const data = await r.json();
if (onUsage && data.usage) {
try { onUsage(data.usage); } catch { /* swallow callback errors */ }
}
return data.content?.[0]?.text || '';
}
// Retry on 5xx and 429; fail fast on 4xx (auth/quota/bad request — retry won't help).
if (r.status >= 500 || r.status === 429) {
lastError = new Error(`Router LLM ${r.status}: ${await r.text()}`);
} else {
const fatal = new Error(`Router LLM ${r.status}: ${await r.text()}`);
fatal.fatal = true;
throw fatal;
}
} catch (err) {
// Re-throw fatal errors (4xx) instead of retrying them.
if (err && err.fatal) { clearTimeout(timer); throw err; }
// Network-level failure (fetch failed / ECONNRESET / TLS / per-attempt timeout). Retry-eligible.
lastError = err;
} finally {
clearTimeout(timer);
}
if (attempt < maxRetries) {
await sleepImpl(retryBaseDelayMs * 2 ** attempt);
}
}
throw lastError;
}
function hashPrompt(s) {
@@ -406,17 +506,18 @@ export async function classify(prompt, registry, options = {}) {
return { ...cache.get(key), source: 'cache' };
}
// Layer 2 — Sonnet 4.6.
// Layer 2 — Sonnet 4.6 with prompt caching (ephemeral 5m TTL on system block).
const llmCall = options.llmCall || (async () => {
const apiKey = process.env.ROUTER_LLM_KEY;
if (!apiKey) return null;
const classifierPrompt = buildClassifierPrompt(prompt, registry, {
const structured = buildClassifierPromptStructured(prompt, registry, {
enrichment: options.enrichment ?? true,
});
const text = await callAnthropicAPI(classifierPrompt, {
const text = await callAnthropicAPI(structured, {
apiKey,
baseUrl: process.env.ROUTER_LLM_BASE_URL || undefined,
model: options.model || CLASSIFIER_MODEL,
onUsage: options.onUsage,
});
return parseClassifierResponse(text);
});