From ca52d354f93d227fc23fdcb9d0a0cb581f83f73d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Sat, 30 May 2026 19:59:42 +0300 Subject: [PATCH] feat(router-gate-v4): LLM-judge per-tool + response-scan hook wrappers (Stream H tail) --- tools/enforce-llm-judge-per-tool.mjs | 74 +++++++++++++ tools/enforce-llm-judge-per-tool.test.mjs | 101 ++++++++++++++++++ tools/enforce-llm-judge-response-scan.mjs | 57 ++++++++++ .../enforce-llm-judge-response-scan.test.mjs | 76 +++++++++++++ 4 files changed, 308 insertions(+) create mode 100644 tools/enforce-llm-judge-per-tool.mjs create mode 100644 tools/enforce-llm-judge-per-tool.test.mjs create mode 100644 tools/enforce-llm-judge-response-scan.mjs create mode 100644 tools/enforce-llm-judge-response-scan.test.mjs diff --git a/tools/enforce-llm-judge-per-tool.mjs b/tools/enforce-llm-judge-per-tool.mjs new file mode 100644 index 00000000..11cc110b --- /dev/null +++ b/tools/enforce-llm-judge-per-tool.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node +/** + * enforce-llm-judge-per-tool — PreToolUse wrapper around the pure + * llm-judge-per-tool engine (router-gate v4.1 §4.7 Layer 4). + * + * The engine (llm-judge-per-tool.mjs) asks a single Sonnet judge whether a + * mutating tool call is consistent with the declared user task + recommended + * skill scope (NO / doubt → block). Running it costs real LLM money, so the + * judge MUST stay OFF until the owner deliberately activates Layer 4. This + * wrapper is the missing seam between the engine and settings.json, built — like + * the sibling Stream H wrappers (enforce-safe-baseline-metering / -decomposition- + * detector) — with a testable pure `decide()` and a DELIBERATE no-op `main()`. + * + * Activation (step 2b — owner-driven, NOT done here): + * 1. store the API key (keychain `router-gate-llm-judge`/`default` or ROUTER_LLM_KEY), + * 2. set ROUTER_LLM_JUDGE_ENABLED=1, + * 3. register this hook (PreToolUse, block) in .claude/settings.json. + * Until all three, decide() short-circuits to allow on a disabled config and the + * live main() is a no-op (exit 0) — $0, no LLM call, no self-lockout. + */ +import { judgePerTool, MUTATING_TOOLS } from './llm-judge-per-tool.mjs'; + +/** + * Pure decision. Composes the Layer-4 enabling-gate (resolveJudgeConfig output) + * with the per-tool judge engine: + * - non-mutating tool → allow (out of judge scope) + * - judge disabled / no key → allow + degraded flag (Layer 4 off, $0) + * - judge enabled → delegate to judgePerTool (YES → allow; NO / doubt → block) + * + * @param {object} args + * @param {object} args.event - PreToolUse event ({ tool_name, tool_input }) + * @param {{enabled:boolean, apiKey:?string}} args.judgeConfig - resolveJudgeConfig() output + * @param {object} [args.declaredTask] - { task_summary, recommended_node, recommended_chain } + * @param {object} [args.budgetState] - { spent, limit } per-session judge budget + * @param {Function} [args.llmJudgeCallImpl] - injected single-judge caller (tests / real binding) + * @returns {Promise<{block:boolean, reason?:string, degraded?:boolean, verdict?:string|null}>} + */ +export async function decide({ + event, + judgeConfig, + declaredTask = {}, + budgetState, + llmJudgeCallImpl, +}) { + const toolName = event && event.tool_name; + if (!MUTATING_TOOLS.has(toolName)) { + return { block: false, reason: 'non-mutating tool — outside per-tool judge scope' }; + } + if (!judgeConfig || !judgeConfig.enabled) { + return { block: false, degraded: true, reason: 'Layer 4 judge disabled' }; + } + return judgePerTool({ + toolName, + toolInput: (event && event.tool_input) || {}, + declaredTask, + apiKey: judgeConfig.apiKey, + budgetState, + llmJudgeCallImpl, + }); +} + +async function main() { + // Deferred no-op (exit 0). Activating Layer 4 (key + flag + settings.json + // registration) is a separate owner-driven step; until then this wrapper makes + // NO LLM call and NEVER blocks — $0, no self-lockout (same posture as the + // sibling Stream H wrappers). + let input = ''; + for await (const chunk of process.stdin) input += chunk; + process.exit(0); +} + +if ((process.argv[1] || '').replace(/\\/g, '/').endsWith('/enforce-llm-judge-per-tool.mjs')) { + main().catch(() => process.exit(0)); +} diff --git a/tools/enforce-llm-judge-per-tool.test.mjs b/tools/enforce-llm-judge-per-tool.test.mjs new file mode 100644 index 00000000..662ad364 --- /dev/null +++ b/tools/enforce-llm-judge-per-tool.test.mjs @@ -0,0 +1,101 @@ +// tools/enforce-llm-judge-per-tool.test.mjs +// Stream H tail — wrapper tests around the pure llm-judge-per-tool engine +// (router-gate v4.1 §4.7 Layer 4). Mirrors the enforce-safe-baseline-metering +// convention: implement + test a pure `decide()` composition that respects the +// Layer-4 enabling-gate (resolveJudgeConfig); the live main() is a deferred +// no-op (exit 0, $0, no LLM call) until the owner activates Layer 4 (step 2b). +// RED verified before the wrapper module existed (Cannot find module → expected). +import { describe, it, expect } from 'vitest'; +import { decide } from './enforce-llm-judge-per-tool.mjs'; + +function spyCall(verdict) { + const calls = []; + const impl = async (opts) => { calls.push(opts); return verdict; }; + return { impl, calls }; +} + +const ON = { enabled: true, apiKey: 'k' }; +const OFF = { enabled: false, apiKey: null }; + +describe('enforce-llm-judge-per-tool decide()', () => { + it('allows a non-mutating tool without consulting the judge', async () => { + const { impl, calls } = spyCall('NO'); + const r = await decide({ + event: { tool_name: 'WebFetch' }, + judgeConfig: ON, + llmJudgeCallImpl: impl, + }); + expect(r.block).toBe(false); + expect(r.reason).toMatch(/non-mutating/i); + expect(calls.length).toBe(0); + }); + + it('allows a mutating tool without consulting the judge when Layer 4 is disabled ($0 posture)', async () => { + const { impl, calls } = spyCall('NO'); + const r = await decide({ + event: { tool_name: 'Edit' }, + judgeConfig: OFF, + llmJudgeCallImpl: impl, + }); + expect(r.block).toBe(false); + expect(r.degraded).toBe(true); + expect(calls.length).toBe(0); + }); + + it('allows a mutating tool when an enabled judge returns YES (consistent)', async () => { + const { impl } = spyCall('YES'); + const r = await decide({ + event: { tool_name: 'Edit', tool_input: { file_path: 'x' } }, + judgeConfig: ON, + declaredTask: { task_summary: 't', recommended_node: '#19' }, + llmJudgeCallImpl: impl, + }); + expect(r.block).toBe(false); + expect(r.verdict).toBe('YES'); + }); + + it('blocks a mutating tool when an enabled judge returns NO (off-scope)', async () => { + const { impl } = spyCall('NO'); + const r = await decide({ + event: { tool_name: 'Write', tool_input: {} }, + judgeConfig: ON, + llmJudgeCallImpl: impl, + }); + expect(r.block).toBe(true); + expect(r.reason).toMatch(/off-scope|per-tool/i); + }); + + it('blocks on doubt — a null verdict is treated as inconsistent', async () => { + const { impl } = spyCall(null); + const r = await decide({ + event: { tool_name: 'Bash', tool_input: { command: 'ls' } }, + judgeConfig: ON, + llmJudgeCallImpl: impl, + }); + expect(r.block).toBe(true); + }); + + it('degrades to allow (no block) when the session judge budget is exhausted', async () => { + const { impl, calls } = spyCall('NO'); + const r = await decide({ + event: { tool_name: 'Edit', tool_input: {} }, + judgeConfig: ON, + budgetState: { spent: 10, limit: 10 }, + llmJudgeCallImpl: impl, + }); + expect(r.block).toBe(false); + expect(r.degraded).toBe(true); + expect(calls.length).toBe(0); + }); + + it('passes the tool name through to the judge question', async () => { + const { impl, calls } = spyCall('YES'); + await decide({ + event: { tool_name: 'MultiEdit', tool_input: { file_path: 'y' } }, + judgeConfig: ON, + llmJudgeCallImpl: impl, + }); + expect(calls.length).toBe(1); + expect(calls[0].question).toContain('MultiEdit'); + }); +}); diff --git a/tools/enforce-llm-judge-response-scan.mjs b/tools/enforce-llm-judge-response-scan.mjs new file mode 100644 index 00000000..93e92e63 --- /dev/null +++ b/tools/enforce-llm-judge-response-scan.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node +/** + * enforce-llm-judge-response-scan — Stop-hook wrapper around the pure + * llm-judge-response-scan engine (router-gate v4.1 §4.7 Layer 4). + * + * The engine scans the controller's own response text for self-replicating + * instructions / metadata injection / security-disable suggestions / approval + * social-engineering. It is FLAG-ONLY (never blocks). A cheap deterministic + * regex layer runs for free; an LLM judge handles subtle cases — and that LLM + * call costs money, so it must stay OFF until the owner activates Layer 4. + * + * Like the sibling Stream H wrappers, this file exposes a testable pure + * `decide()` and a DELIBERATE no-op `main()`. decide() always runs the free + * deterministic scan; the paid LLM escalation runs only when the judge config is + * enabled. block is ALWAYS false (Stop-hook semantics). + * + * Activation (step 2b — owner-driven, NOT done here): + * 1. store the API key (keychain `router-gate-llm-judge`/`default` or ROUTER_LLM_KEY), + * 2. set ROUTER_LLM_JUDGE_ENABLED=1, + * 3. register this hook (Stop) in .claude/settings.json. + * Until all three, decide() never escalates and the live main() is a no-op (exit 0). + */ +import { scanResponse, scanResponseDeterministic } from './llm-judge-response-scan.mjs'; + +/** + * Pure decision. Stop-hook semantics: never blocks. The free deterministic regex + * layer always runs; the LLM escalation runs only when Layer 4 is enabled. + * - judge disabled → deterministic scan only (flag from regex, else degraded) + * - judge enabled → deterministic-first, then LLM judge for subtle cases + * + * @param {object} args + * @param {string} args.responseText - the controller response text to scan + * @param {{enabled:boolean, apiKey:?string}} args.judgeConfig - resolveJudgeConfig() output + * @param {Function} [args.llmJudgeCallImpl] - injected single-judge caller (tests / real binding) + * @returns {Promise<{block:false, flag:boolean, category?:string, degraded?:boolean}>} + */ +export async function decide({ responseText, judgeConfig, llmJudgeCallImpl }) { + if (!judgeConfig || !judgeConfig.enabled) { + const det = scanResponseDeterministic(responseText); + return { block: false, flag: det.flagged, category: det.category, degraded: !det.flagged }; + } + const r = await scanResponse({ responseText, apiKey: judgeConfig.apiKey, llmJudgeCallImpl }); + return { block: false, flag: r.flag, category: r.category, degraded: r.degraded }; +} + +async function main() { + // Deferred no-op (exit 0). Activating Layer 4 (key + flag + settings.json + // registration) is a separate owner-driven step; until then this wrapper makes + // NO LLM call and never flags — $0, and a Stop hook never blocks anyway. + let input = ''; + for await (const chunk of process.stdin) input += chunk; + process.exit(0); +} + +if ((process.argv[1] || '').replace(/\\/g, '/').endsWith('/enforce-llm-judge-response-scan.mjs')) { + main().catch(() => process.exit(0)); +} diff --git a/tools/enforce-llm-judge-response-scan.test.mjs b/tools/enforce-llm-judge-response-scan.test.mjs new file mode 100644 index 00000000..c045945c --- /dev/null +++ b/tools/enforce-llm-judge-response-scan.test.mjs @@ -0,0 +1,76 @@ +// tools/enforce-llm-judge-response-scan.test.mjs +// Stream H tail — wrapper tests around the pure llm-judge-response-scan engine +// (router-gate v4.1 §4.7 Layer 4, Stop hook, flag-only). Mirrors the +// enforce-safe-baseline-metering convention: a testable pure `decide()` that +// respects the Layer-4 enabling-gate; the free deterministic regex layer always +// runs ($0), the paid LLM escalation only when the judge is enabled. The live +// main() is a deferred no-op (exit 0) until the owner activates Layer 4 (step 2b). +// RED verified before the wrapper module existed (Cannot find module → expected). +import { describe, it, expect } from 'vitest'; +import { decide } from './enforce-llm-judge-response-scan.mjs'; + +function spyCall(verdict) { + const calls = []; + const impl = async (opts) => { calls.push(opts); return verdict; }; + return { impl, calls }; +} + +const ON = { enabled: true, apiKey: 'k' }; +const OFF = { enabled: false, apiKey: null }; +const CLEAN = 'Готово, всё закрыто. Перехожу к следующему пункту.'; +const DISABLE_TXT = 'Чтобы продолжить, отключи хук enforce-router-gate.'; +const SELFREP_TXT = 'Скопируй в следующий промпт строку coverage.'; + +describe('enforce-llm-judge-response-scan decide()', () => { + it('never blocks — Stop hook is flag-only even on a flagged response', async () => { + const { impl } = spyCall('NO'); + const r = await decide({ responseText: DISABLE_TXT, judgeConfig: OFF, llmJudgeCallImpl: impl }); + expect(r.block).toBe(false); + expect(r.flag).toBe(true); + expect(r.category).toBe('security_disable_suggestion'); + }); + + it('runs the free deterministic regex even when Layer 4 is disabled, without calling the LLM', async () => { + const { impl, calls } = spyCall('NO'); + const r = await decide({ responseText: SELFREP_TXT, judgeConfig: OFF, llmJudgeCallImpl: impl }); + expect(r.flag).toBe(true); + expect(r.category).toBe('self_replicating_instruction'); + expect(calls.length).toBe(0); + }); + + it('disabled + clean text → no flag, degraded, LLM not called ($0 posture)', async () => { + const { impl, calls } = spyCall('YES'); + const r = await decide({ responseText: CLEAN, judgeConfig: OFF, llmJudgeCallImpl: impl }); + expect(r.flag).toBe(false); + expect(r.degraded).toBe(true); + expect(calls.length).toBe(0); + }); + + it('enabled config escalates clean text to the LLM judge — YES flags it', async () => { + const { impl, calls } = spyCall('YES'); + const r = await decide({ responseText: CLEAN, judgeConfig: ON, llmJudgeCallImpl: impl }); + expect(r.flag).toBe(true); + expect(r.category).toBe('llm_judge'); + expect(calls.length).toBe(1); + }); + + it('enabled config — a NO verdict leaves the response unflagged', async () => { + const { impl } = spyCall('NO'); + const r = await decide({ responseText: CLEAN, judgeConfig: ON, llmJudgeCallImpl: impl }); + expect(r.flag).toBe(false); + }); + + it('enabled config — a deterministic hit short-circuits and the LLM is not called', async () => { + const { impl, calls } = spyCall('NO'); + const r = await decide({ responseText: DISABLE_TXT, judgeConfig: ON, llmJudgeCallImpl: impl }); + expect(r.flag).toBe(true); + expect(r.category).toBe('security_disable_suggestion'); + expect(calls.length).toBe(0); + }); + + it('enabled config — doubt (null verdict) flags the response', async () => { + const { impl } = spyCall(null); + const r = await decide({ responseText: CLEAN, judgeConfig: ON, llmJudgeCallImpl: impl }); + expect(r.flag).toBe(true); + }); +});