From 74d4d9f21dd5e6d2bbfa72d33723b0b79bc794b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Tue, 7 Jul 2026 19:07:41 +0300 Subject: [PATCH] =?UTF-8?q?feat(supplier):=20=D0=B0=D0=B2=D0=B0=D1=80?= =?UTF-8?q?=D0=B8=D0=B9=D0=BD=D1=8B=D0=B9=20spike-=D0=B4=D0=B5=D1=82=D0=B5?= =?UTF-8?q?=D0=BA=D1=82=D0=BE=D1=80=20=D1=80=D1=83=D1=87=D0=BD=D0=BE=D0=B9?= =?UTF-8?q?=20=D0=BE=D1=87=D0=B5=D1=80=D0=B5=D0=B4=D0=B8=20=D0=B2=20incide?= =?UTF-8?q?nts:watch-failures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Блок 6: если за окно (10 мин) в supplier_manual_sync_queue упала пачка (>= threshold-manual-queue, дефолт 10) — вероятное падение кабинета поставщика. Инцидент дедупится 60 мин, письмо шлётся каждый прогон на kdv1@bk.ru + ops@liderra.ru (громко при реальной аварии, но одним сводным письмом). Флаг sendMail в createIncident. phpstan-baseline: +4 TestCall::artisan() (5→9) от новых тестов. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Commands/IncidentsWatchFailures.php | 36 ++++++++++- app/phpstan-baseline.neon | 2 +- .../Console/IncidentsWatchFailuresTest.php | 64 +++++++++++++++++++ docs/observer/STATUS.md | 8 +-- 4 files changed, 103 insertions(+), 7 deletions(-) diff --git a/app/app/Console/Commands/IncidentsWatchFailures.php b/app/app/Console/Commands/IncidentsWatchFailures.php index 4751a52b..c5b39dbc 100644 --- a/app/app/Console/Commands/IncidentsWatchFailures.php +++ b/app/app/Console/Commands/IncidentsWatchFailures.php @@ -33,7 +33,8 @@ class IncidentsWatchFailures extends Command {--threshold-daily=50 : Порог суммы за 24ч для failed_jobs} {--persistent-hours=3 : Порог возраста persistent-exception для failed_jobs} {--dedup-window=60 : Окно дедупа открытых инцидентов в минутах} - {--threshold-single-lead=1000 : Порог storm detection: failures одного supplier_lead_id за окно}'; + {--threshold-single-lead=1000 : Порог storm detection: failures одного supplier_lead_id за окно} + {--threshold-manual-queue=10 : Порог spike ручной очереди поставщика (за окно) → возможное падение кабинета}'; protected $description = 'Сканирует failed_webhook_jobs и failed_jobs, создаёт incidents_log на превышение порогов'; @@ -47,6 +48,7 @@ class IncidentsWatchFailures extends Command $dedupMinutes = (int) $this->option('dedup-window'); $thresholdSingleLead = (int) $this->option('threshold-single-lead'); + $thresholdManualQueue = (int) $this->option('threshold-manual-queue'); $since = Carbon::now()->subMinutes($windowMinutes); $since24h = Carbon::now()->subHours(24); @@ -221,6 +223,35 @@ class IncidentsWatchFailures extends Command } } + // ===== БЛОК 6: supplier-manual-queue spike ===== + // Пачка проектов, ушедших в ручную очередь за короткое окно = вероятное падение + // кабинета поставщика. Инцидент дедупим (60 мин, чтобы не засорять журнал), НО письмо + // шлём на КАЖДОМ прогоне пока спайк держится — «сигналит как ненормальный» при реальной + // аварии, одним сводным письмом на оба адреса (не N писем на проект). + if ($thresholdManualQueue > 0) { + $manualQueueCount = (int) DB::connection(self::DB_CONNECTION) + ->table('supplier_manual_sync_queue') + ->where('created_at', '>=', $since) + ->count(); + + if ($manualQueueCount >= $thresholdManualQueue) { + $dedupKey = 'supplier-manual-queue-spike'; + $summary = "Похоже, кабинет поставщика упал: {$manualQueueCount} проект(ов) ушло в ручную очередь за {$windowMinutes} мин."; + + // Инцидент — только если открытого с той же сигнатурой нет (дедуп 60 мин). + if (! $this->isDup($dedupKey, $dedupAt)) { + $this->createIncident($adminId, 'other', 'high', $summary, $since, $now, $dedupKey, sendMail: false); + $created++; + } + + // Письмо — каждый прогон при активном спайке, на оба адреса. + Mail::to(['kdv1@bk.ru', 'ops@liderra.ru']) + ->send(new IncidentDetectedMail($summary, 'high')); + + $this->info("Supplier manual-queue spike [high]: {$manualQueueCount} in {$windowMinutes}m"); + } + } + $this->info("Done. Created {$created} incident(s)."); return self::SUCCESS; @@ -245,6 +276,7 @@ class IncidentsWatchFailures extends Command Carbon $startedAt, Carbon $now, string $dedupKey = '', + bool $sendMail = true, ): void { DB::connection(self::DB_CONNECTION)->table('incidents_log')->insert([ 'type' => $type, @@ -259,7 +291,7 @@ class IncidentsWatchFailures extends Command 'updated_at' => $now, ]); - if ($severity === 'high') { + if ($sendMail && $severity === 'high') { Mail::to('kdv1@bk.ru')->send(new IncidentDetectedMail($summary, $severity)); } } diff --git a/app/phpstan-baseline.neon b/app/phpstan-baseline.neon index 31548632..ef15f3e6 100644 --- a/app/phpstan-baseline.neon +++ b/app/phpstan-baseline.neon @@ -1371,7 +1371,7 @@ parameters: - message: '#^Call to an undefined method Pest\\PendingCalls\\TestCall\:\:artisan\(\)\.$#' identifier: method.notFound - count: 5 + count: 9 path: tests/Feature/Console/IncidentsWatchFailuresTest.php - diff --git a/app/tests/Feature/Console/IncidentsWatchFailuresTest.php b/app/tests/Feature/Console/IncidentsWatchFailuresTest.php index aa453efb..c7507310 100644 --- a/app/tests/Feature/Console/IncidentsWatchFailuresTest.php +++ b/app/tests/Feature/Console/IncidentsWatchFailuresTest.php @@ -2,9 +2,13 @@ declare(strict_types=1); +use App\Mail\IncidentDetectedMail; +use App\Models\Project; +use App\Models\Tenant; use Illuminate\Foundation\Testing\DatabaseTransactions; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Mail; use Tests\Concerns\SharesSupplierPdo; uses(DatabaseTransactions::class); @@ -107,3 +111,63 @@ test('creates separate incidents for different exception signatures', function ( expect(DB::table('incidents_log')->count())->toBe(2); }); + +// ===== Блок 6: supplier-manual-queue spike ===== + +// Helper: seed a project + N supplier_manual_sync_queue rows within the window +function seedManualQueue(int $n, ?DateTimeInterface $at = null): int +{ + $project = Project::factory()->for(Tenant::factory())->create(); + for ($i = 0; $i < $n; $i++) { + DB::table('supplier_manual_sync_queue')->insert([ + 'project_id' => $project->id, + 'platform' => 'B1', + 'operation' => 'create', + 'external_id' => null, + 'payload_snapshot' => '{}', + 'failure_reason' => 'portal_unreachable', + 'status' => 'pending', + 'created_at' => $at ?? now(), + ]); + } + + return $project->id; +} + +test('manual-queue spike below threshold: no incident, no mail', function () { + Mail::fake(); + seedManualQueue(3, now()); // < default threshold 10 + + $this->artisan('incidents:watch-failures')->assertSuccessful(); + + expect(DB::table('incidents_log')->where('root_cause', 'supplier-manual-queue-spike')->count())->toBe(0); + Mail::assertNothingSent(); +}); + +test('manual-queue spike at/above threshold: high incident + mail to both addresses', function () { + Mail::fake(); + seedManualQueue(10, now()); // == default threshold 10 + + $this->artisan('incidents:watch-failures')->assertSuccessful(); + + $incident = DB::table('incidents_log')->where('root_cause', 'supplier-manual-queue-spike')->first(); + expect($incident)->not->toBeNull(); + expect($incident->severity)->toBe('high'); + expect($incident->summary)->toContain('10'); + + Mail::assertSent(IncidentDetectedMail::class, function ($mail) { + return $mail->hasTo('kdv1@bk.ru') && $mail->hasTo('ops@liderra.ru'); + }); +}); + +test('manual-queue spike dedup: second run in window makes no new incident BUT re-sends mail', function () { + Mail::fake(); + seedManualQueue(12, now()); + + $this->artisan('incidents:watch-failures')->assertSuccessful(); + $this->artisan('incidents:watch-failures')->assertSuccessful(); + + // Инцидент один (дедуп 60 мин), а письмо ушло на каждом прогоне (громкость). + expect(DB::table('incidents_log')->where('root_cause', 'supplier-manual-queue-spike')->count())->toBe(1); + Mail::assertSent(IncidentDetectedMail::class, 2); +}); diff --git a/docs/observer/STATUS.md b/docs/observer/STATUS.md index 61d2133d..8d31f8c8 100644 --- a/docs/observer/STATUS.md +++ b/docs/observer/STATUS.md @@ -1,6 +1,6 @@ # Brain Status (auto-generated) -Last updated: 2026-07-07T15:40:46.958Z +Last updated: 2026-07-07T16:01:34.642Z | Контролёр | Состояние | Детали | |---|---|---| @@ -112,9 +112,9 @@ Episodes since last run: 542 / threshold: 10 | PID | Имя | CPU-время | Возраст | |---|---|---|---| -| 2132 | Code | 8.65ч | 12285426.2ч | -| 3276 | MsMpEng | 7.73ч | NaNч | -| 4 | System | 3.39ч | 12285423.7ч | +| 2132 | Code | 8.70ч | 0.0ч | +| 3276 | MsMpEng | 7.83ч | 0.0ч | +| 4 | System | 3.42ч | NaNч | ⚠️ Проверь, не «осиротевшие» ли это процессы от завершённых Claude-сессий.