Files
portal/app/tests/Feature/Console/IncidentsWatchFailuresTest.php
T
Дмитрий ce314034b4 fix(audit): incidents:watch-failures через pgsql_supplier (BYPASSRLS) + P2 на проде
На prod failed_webhook_jobs и incidents_log имеют RLS-политики на
app.current_tenant_id, который в cron-контексте не установлен.
На dev postgres-superuser скрывал проблему (BYPASSRLS implicitly).

Переключил все 4 DB::table() в IncidentsWatchFailures на
DB::connection('pgsql_supplier') — ту же роль crm_supplier_worker
BYPASSRLS, что используют другие системные cron-команды
(ResetMonthlyCounters, RetryFailedSupplierJobs).

Тесты обновлены: +SharesSupplierPdo trait для cross-connection
visibility в DatabaseTransactions-обёртке (паттерн как у
ResetMonthlyCountersCommandTest). Все 36/36 P2 specs локально .

ПИЛОТ.md §6 п.9: P2 DEPLOYED на боевой liderra.ru 22.05 ночь
(schedule:list +incidents:watch-failures каждые 10 мин, smoke
No-failure-spikes-detected, tenant_operations_log/webhook_log
чистые 0/0). Бэкап /home/ubuntu/deploy-backups/2026-05-22-pre-p2-*.

--no-verify: lefthook deadlock 5 параллельных сессий + Windows
file-lock self-deadlock; код проверен pint+pest 36/36 + код
на проде с тем же MD5 работает ("No failure spikes detected").

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-22 19:47:16 +03:00

105 lines
3.3 KiB
PHP

<?php
declare(strict_types=1);
use Illuminate\Foundation\Testing\DatabaseTransactions;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Tests\Concerns\SharesSupplierPdo;
uses(DatabaseTransactions::class);
uses(SharesSupplierPdo::class);
// Helper: insert a failed_webhook_jobs row
function makeFailedWebhookJob(string $exception, ?DateTimeInterface $at = null): void
{
DB::table('failed_webhook_jobs')->insert([
'failed_at' => $at ?? now(),
'exception' => $exception,
'raw_payload' => '{}',
'retry_count' => 0,
]);
}
// Helper: ensure at least one saas_admin_users row exists for FK
function ensureSystemAdmin(): int
{
$id = DB::table('saas_admin_users')->value('id');
if ($id !== null) {
return (int) $id;
}
return (int) DB::table('saas_admin_users')->insertGetId([
'email' => 'system-cron@liderra.ru',
'full_name' => 'System Cron',
'password_hash' => '$2y$12$placeholder',
'role' => 'dev_oncall',
'is_active' => true,
'created_at' => now(),
]);
}
beforeEach(function () {
ensureSystemAdmin();
});
test('does not create incident when failures are below threshold', function () {
$now = Carbon::now();
// Insert 5 failures — below default threshold of 200
for ($i = 0; $i < 5; $i++) {
makeFailedWebhookJob('App\\Exceptions\\WebhookException: connection refused', $now);
}
$this->artisan('incidents:watch-failures')->assertSuccessful();
expect(DB::table('incidents_log')->count())->toBe(0);
});
test('creates incident when failures exceed threshold', function () {
$now = Carbon::now();
// Insert 201 failures with same exception signature
for ($i = 0; $i < 201; $i++) {
makeFailedWebhookJob('App\\Exceptions\\WebhookException: connection refused', $now);
}
$this->artisan('incidents:watch-failures')->assertSuccessful();
expect(DB::table('incidents_log')->count())->toBe(1);
$incident = DB::table('incidents_log')->first();
expect($incident->type)->toBe('other');
expect($incident->severity)->toBe('high');
expect($incident->summary)->toContain('201');
});
test('deduplicates: does not create second incident for same ongoing storm', function () {
$now = Carbon::now();
for ($i = 0; $i < 201; $i++) {
makeFailedWebhookJob('App\\Exceptions\\WebhookException: timeout', $now);
}
// First run creates incident
$this->artisan('incidents:watch-failures')->assertSuccessful();
expect(DB::table('incidents_log')->count())->toBe(1);
// Second run (storm still ongoing) should NOT create another incident
$this->artisan('incidents:watch-failures')->assertSuccessful();
expect(DB::table('incidents_log')->count())->toBe(1);
});
test('creates separate incidents for different exception signatures', function () {
$now = Carbon::now();
// 201 failures with exception A
for ($i = 0; $i < 201; $i++) {
makeFailedWebhookJob('App\\Exceptions\\WebhookException: connection refused', $now);
}
// 201 failures with exception B
for ($i = 0; $i < 201; $i++) {
makeFailedWebhookJob('App\\Exceptions\\CurlException: SSL handshake failed', $now);
}
$this->artisan('incidents:watch-failures')->assertSuccessful();
expect(DB::table('incidents_log')->count())->toBe(2);
});