From 1b40f2d861cafd505c0991e337e3cea27456c3ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Thu, 25 Jun 2026 14:36:31 +0300 Subject: [PATCH] =?UTF-8?q?feat(secretary):=20worker=20foundations=20?= =?UTF-8?q?=E2=80=94=20lock,=20queue,=20spawn,=20input-cap,=20safe-index,?= =?UTF-8?q?=20gitignore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +++ tools/secretary-harvest.mjs | 11 ++++++++- tools/secretary-harvest.test.mjs | 18 ++++++++++++++ tools/secretary-index.mjs | 16 +++++++++++++ tools/secretary-index.test.mjs | 23 ++++++++++++++++-- tools/secretary-lock.mjs | 26 ++++++++++++++++++++ tools/secretary-lock.test.mjs | 26 ++++++++++++++++++++ tools/secretary-queue.mjs | 41 ++++++++++++++++++++++++++++++++ tools/secretary-queue.test.mjs | 30 +++++++++++++++++++++++ tools/secretary-spawn.mjs | 15 ++++++++++++ tools/secretary-spawn.test.mjs | 17 +++++++++++++ 11 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 tools/secretary-lock.mjs create mode 100644 tools/secretary-lock.test.mjs create mode 100644 tools/secretary-queue.mjs create mode 100644 tools/secretary-queue.test.mjs create mode 100644 tools/secretary-spawn.mjs create mode 100644 tools/secretary-spawn.test.mjs diff --git a/.gitignore b/.gitignore index b7c47be..4b84bc2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ node_modules/ .DS_Store graphify-out/ graphify-out-*/ + +# Secretary background worker scratch (queue / lock / cursor) — per theme, never committed +docs/secretary/*/_worker/ diff --git a/tools/secretary-harvest.mjs b/tools/secretary-harvest.mjs index 56fc8a1..075f4f8 100644 --- a/tools/secretary-harvest.mjs +++ b/tools/secretary-harvest.mjs @@ -87,9 +87,18 @@ export function trunkForCatcher(proto) { return `РЕШЕНИЯ СТВОЛА (уже решено — НЕ лови):\n${d}\nОТКРЫТЫЕ СТВОЛА (уже на виду — НЕ лови):\n${o}`; } +/** Лимит на размер выдачи действия в промпте. По умолчанию полный (Infinity) — владелец хочет видеть + * тяжёлый research целиком; фон-воркер без таймаут-стены это выдерживает. Env урезает при желании. */ +function inputCap() { const v = process.env.SECRETARY_INPUT_CAP; const n = Number(v); return v && Number.isFinite(n) ? n : Infinity; } +function clamp(s) { + const cap = inputCap(); + if (!Number.isFinite(cap) || s.length <= cap) return s; + return s.slice(0, cap) + `…[вырезано ${s.length - cap} знаков]`; +} + /** Текст обмена из спана {user, assistant, actions} (заменяет exchange(span) песочницы). */ export function renderExchangeText(spanEx) { - const acts = ((spanEx.actions || []).map((a) => ` • ${a.tool} in=${a.input ?? ''}${a.result != null ? ` → ${String(a.result).slice(0, 4000)}` : ''}`).join('\n')) || '—'; + const acts = ((spanEx.actions || []).map((a) => ` • ${a.tool} in=${a.input ?? ''}${a.result != null ? ` → ${clamp(String(a.result))}` : ''}`).join('\n')) || '—'; return `[ЮЗЕР]: ${spanEx.user || ''}\n[АССИСТЕНТ]: ${spanEx.assistant || ''}\n[ДЕЙСТВИЯ]:\n${acts}`; } diff --git a/tools/secretary-harvest.test.mjs b/tools/secretary-harvest.test.mjs index a6546b4..672349c 100644 --- a/tools/secretary-harvest.test.mjs +++ b/tools/secretary-harvest.test.mjs @@ -60,3 +60,21 @@ describe('USER-сборщики', () => { expect(u).toContain('находка'); }); }); + +describe('renderExchangeText input cap', () => { + const bigSpan = () => ({ user: 'u', assistant: 'a', actions: [{ tool: 'perplexity', input: 'q', result: 'X'.repeat(9000) }] }); + it('default: no truncation — full result passes through', () => { + delete process.env.SECRETARY_INPUT_CAP; + const t = renderExchangeText(bigSpan()); + expect(t).toContain('X'.repeat(9000)); + expect(t).not.toContain('вырезано'); + }); + it('finite cap truncates the result with a marker', () => { + process.env.SECRETARY_INPUT_CAP = '100'; + const t = renderExchangeText(bigSpan()); + expect(t).toContain('X'.repeat(100)); + expect(t).not.toContain('X'.repeat(101)); + expect(t).toMatch(/вырезано \d+ знаков/); + delete process.env.SECRETARY_INPUT_CAP; + }); +}); diff --git a/tools/secretary-index.mjs b/tools/secretary-index.mjs index 64d5ebd..c050798 100644 --- a/tools/secretary-index.mjs +++ b/tools/secretary-index.mjs @@ -1,3 +1,7 @@ +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import lockfile from 'proper-lockfile'; +import { writeFileAtomic } from './secretary-layer1.mjs'; + // Апсерт строки дела в оглавление (§D8). Ключ — /protocol.md. export function upsertIndexEntry(indexMd, { slug, title, goal, status, date }) { const line = `- [${title}](${slug}/protocol.md) — ${goal} · ${status} · ${date}`; @@ -8,3 +12,15 @@ export function upsertIndexEntry(indexMd, { slug, title, goal, status, date }) { else lines.push(line); return lines.join('\n'); } + +/** Безопасная к параллели запись строки темы в оглавление: read-modify-write под замком файла. + * Две темы пишут одно оглавление — замок сериализует, upsert по ключу-теме не теряет строк. + * proper-lockfile стат-ит цель → файл должен существовать ДО взятия замка (создаём пустой). */ +export async function writeIndexSafely(idxFile, entry, { lockImpl = lockfile } = {}) { + if (!existsSync(idxFile)) { try { writeFileSync(idxFile, ''); } catch { /* гонка создания — переживём */ } } + const release = await lockImpl.lock(idxFile, { stale: 15_000, retries: { retries: 10, minTimeout: 20, maxTimeout: 200 }, realpath: false }); + try { + const cur = existsSync(idxFile) ? readFileSync(idxFile, 'utf-8') : ''; + writeFileAtomic(idxFile, upsertIndexEntry(cur, entry)); + } finally { await release(); } +} diff --git a/tools/secretary-index.test.mjs b/tools/secretary-index.test.mjs index 22c3ef2..4a93bee 100644 --- a/tools/secretary-index.test.mjs +++ b/tools/secretary-index.test.mjs @@ -1,5 +1,8 @@ -import { describe, it, expect } from 'vitest'; -import { upsertIndexEntry } from './secretary-index.mjs'; +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { upsertIndexEntry, writeIndexSafely } from './secretary-index.mjs'; describe('upsertIndexEntry', () => { it('добавляет новое дело', () => { @@ -14,3 +17,19 @@ describe('upsertIndexEntry', () => { expect(upd).toContain('закрыто'); }); }); + +describe('writeIndexSafely', () => { + let dir, idx; + beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'secidx-')); idx = join(dir, 'содержание.md'); writeFileSync(idx, ''); }); + afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + + it('two concurrent upserts of different themes both survive', async () => { + await Promise.all([ + writeIndexSafely(idx, { slug: 'alpha', title: 'alpha', goal: 'g', status: 'открыто', date: 'd' }), + writeIndexSafely(idx, { slug: 'beta', title: 'beta', goal: 'g', status: 'открыто', date: 'd' }), + ]); + const md = readFileSync(idx, 'utf-8'); + expect(md).toContain('(alpha/protocol.md)'); + expect(md).toContain('(beta/protocol.md)'); + }); +}); diff --git a/tools/secretary-lock.mjs b/tools/secretary-lock.mjs new file mode 100644 index 0000000..9822c59 --- /dev/null +++ b/tools/secretary-lock.mjs @@ -0,0 +1,26 @@ +// tools/secretary-lock.mjs +// Замок темы поверх proper-lockfile: один воркер на тему. Авто-обновление mtime (update) = пульс, +// устаревание (stale) = перехват мёртвого. Цель замка — папка _worker внутри темы. +import lockfile from 'proper-lockfile'; +import { join } from 'node:path'; + +const STALE_MS = Number(process.env.SECRETARY_LOCK_STALE_MS) || 30_000; +const UPDATE_MS = Number(process.env.SECRETARY_LOCK_UPDATE_MS) || 10_000; + +function target(workDir) { return join(workDir, '_worker'); } + +/** Взять замок темы. По умолчанию кидает при занятом (ELOCKED). graceful:true → вернуть null вместо броска. */ +export async function acquireThemeLock(workDir, { graceful = false } = {}) { + try { + return await lockfile.lock(target(workDir), { stale: STALE_MS, update: UPDATE_MS, realpath: false }); + } catch (e) { + if (graceful) return null; + throw e; + } +} + +/** Занят ли замок темы живым держателем. */ +export async function isThemeLocked(workDir) { + try { return await lockfile.check(target(workDir), { stale: STALE_MS, realpath: false }); } + catch { return false; } +} diff --git a/tools/secretary-lock.test.mjs b/tools/secretary-lock.test.mjs new file mode 100644 index 0000000..b93e3a9 --- /dev/null +++ b/tools/secretary-lock.test.mjs @@ -0,0 +1,26 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { acquireThemeLock, isThemeLocked } from './secretary-lock.mjs'; + +let dir; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'seclock-')); mkdirSync(join(dir, '_worker'), { recursive: true }); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +describe('secretary-lock', () => { + it('first acquire succeeds and returns a release fn; second acquire fails while held', async () => { + const release = await acquireThemeLock(dir); + expect(typeof release).toBe('function'); + expect(await isThemeLocked(dir)).toBe(true); + await expect(acquireThemeLock(dir)).rejects.toBeTruthy(); + await release(); + expect(await isThemeLocked(dir)).toBe(false); + }); + it('acquire returns null instead of throwing when graceful:true and already held', async () => { + const release = await acquireThemeLock(dir); + const second = await acquireThemeLock(dir, { graceful: true }); + expect(second).toBe(null); + await release(); + }); +}); diff --git a/tools/secretary-queue.mjs b/tools/secretary-queue.mjs new file mode 100644 index 0000000..9e3276c --- /dev/null +++ b/tools/secretary-queue.mjs @@ -0,0 +1,41 @@ +// tools/secretary-queue.mjs +// Очередь спанов и курсор обработки темы на диске (/_worker/). Дедуп по span.index. +// Курсор — наибольший ПОЛНОСТЬЮ обработанный индекс; пишет только воркер. Все записи атомарны. +import { readFileSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { writeFileAtomic } from './secretary-layer1.mjs'; + +function wdir(workDir) { const d = join(workDir, '_worker'); mkdirSync(d, { recursive: true }); return d; } +function qpath(workDir) { return join(wdir(workDir), 'queue.json'); } +function cpath(workDir) { return join(wdir(workDir), 'cursor.json'); } + +function readQueue(workDir) { + try { const a = JSON.parse(readFileSync(qpath(workDir), 'utf-8')); return Array.isArray(a) ? a : []; } + catch { return []; } +} +function writeQueue(workDir, arr) { writeFileAtomic(qpath(workDir), JSON.stringify(arr)); } + +/** Поставить спан в очередь. Дедуп по span.index — повторная постановка того же спана игнорируется. */ +export function enqueueSpan(workDir, job) { + const arr = readQueue(workDir); + const idx = job && job.span ? job.span.index : undefined; + if (arr.some((j) => j.span && j.span.index === idx)) return; + arr.push(job); + writeQueue(workDir, arr); +} + +/** Снять первый спан (FIFO). null если очередь пуста. */ +export function dequeueSpan(workDir) { + const arr = readQueue(workDir); + if (!arr.length) return null; + const first = arr.shift(); + writeQueue(workDir, arr); + return first; +} + +/** Курсор обработки темы (наибольший обработанный индекс). -1 = ничего не обработано. */ +export function readCursor(workDir) { + try { const v = JSON.parse(readFileSync(cpath(workDir), 'utf-8')); return Number.isFinite(v.index) ? v.index : -1; } + catch { return -1; } +} +export function writeCursor(workDir, index) { writeFileAtomic(cpath(workDir), JSON.stringify({ index })); } diff --git a/tools/secretary-queue.test.mjs b/tools/secretary-queue.test.mjs new file mode 100644 index 0000000..6c18f72 --- /dev/null +++ b/tools/secretary-queue.test.mjs @@ -0,0 +1,30 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { enqueueSpan, dequeueSpan, readCursor, writeCursor } from './secretary-queue.mjs'; + +let dir; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'secq-')); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +const job = (index) => ({ session: 's', span: { start: index * 10, end: index * 10 + 5, index, note: '' }, kind: 'span' }); + +describe('secretary-queue', () => { + it('enqueue then dequeue returns jobs in FIFO order', () => { + enqueueSpan(dir, job(0)); enqueueSpan(dir, job(1)); + expect(dequeueSpan(dir).span.index).toBe(0); + expect(dequeueSpan(dir).span.index).toBe(1); + expect(dequeueSpan(dir)).toBe(null); + }); + it('enqueue dedups by span index (no duplicate jobs)', () => { + enqueueSpan(dir, job(2)); enqueueSpan(dir, job(2)); + expect(dequeueSpan(dir).span.index).toBe(2); + expect(dequeueSpan(dir)).toBe(null); + }); + it('cursor defaults to -1 and round-trips', () => { + expect(readCursor(dir)).toBe(-1); + writeCursor(dir, 4); + expect(readCursor(dir)).toBe(4); + }); +}); diff --git a/tools/secretary-spawn.mjs b/tools/secretary-spawn.mjs new file mode 100644 index 0000000..43afa6d --- /dev/null +++ b/tools/secretary-spawn.mjs @@ -0,0 +1,15 @@ +// tools/secretary-spawn.mjs +// «Выстрел» фон-воркера: detached + unref → процесс переживает выход хука, хук его не ждёт. +// Идемпотентность по факту: если воркер уже крутится, новый упрётся в замок темы и выйдет. +import { spawn as realSpawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const here = dirname(fileURLToPath(import.meta.url)); +const WORKER = join(here, 'secretary-worker.mjs'); + +export function spawnWorker(workDir, { spawnImpl = realSpawn } = {}) { + const child = spawnImpl(process.execPath, [WORKER, workDir], { detached: true, stdio: 'ignore' }); + if (child && typeof child.unref === 'function') child.unref(); + return child; +} diff --git a/tools/secretary-spawn.test.mjs b/tools/secretary-spawn.test.mjs new file mode 100644 index 0000000..8150a11 --- /dev/null +++ b/tools/secretary-spawn.test.mjs @@ -0,0 +1,17 @@ +import { describe, it, expect } from 'vitest'; +import { spawnWorker } from './secretary-spawn.mjs'; + +describe('secretary-spawn', () => { + it('spawns node with [worker, workDir], detached, stdio ignore, and unrefs', () => { + const calls = []; + let unrefed = false; + const fakeSpawn = (cmd, args, opts) => { calls.push({ cmd, args, opts }); return { unref: () => { unrefed = true; } }; }; + spawnWorker('C:/themes/alpha', { spawnImpl: fakeSpawn }); + expect(calls).toHaveLength(1); + expect(calls[0].args[1]).toBe('C:/themes/alpha'); + expect(calls[0].args[0]).toMatch(/secretary-worker\.mjs$/); + expect(calls[0].opts.detached).toBe(true); + expect(calls[0].opts.stdio).toBe('ignore'); + expect(unrefed).toBe(true); + }); +});