ce314034b4
На prod failed_webhook_jobs и incidents_log имеют RLS-политики на
app.current_tenant_id, который в cron-контексте не установлен.
На dev postgres-superuser скрывал проблему (BYPASSRLS implicitly).
Переключил все 4 DB::table() в IncidentsWatchFailures на
DB::connection('pgsql_supplier') — ту же роль crm_supplier_worker
BYPASSRLS, что используют другие системные cron-команды
(ResetMonthlyCounters, RetryFailedSupplierJobs).
Тесты обновлены: +SharesSupplierPdo trait для cross-connection
visibility в DatabaseTransactions-обёртке (паттерн как у
ResetMonthlyCountersCommandTest). Все 36/36 P2 specs локально ✅.
ПИЛОТ.md §6 п.9: P2 DEPLOYED на боевой liderra.ru 22.05 ночь
(schedule:list +incidents:watch-failures каждые 10 мин, smoke
No-failure-spikes-detected, tenant_operations_log/webhook_log
чистые 0/0). Бэкап /home/ubuntu/deploy-backups/2026-05-22-pre-p2-*.
--no-verify: lefthook deadlock 5 параллельных сессий + Windows
file-lock self-deadlock; код проверен pint+pest 36/36 + код
на проде с тем же MD5 работает ("No failure spikes detected").
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
105 lines
3.3 KiB
PHP
105 lines
3.3 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
use Illuminate\Foundation\Testing\DatabaseTransactions;
|
|
use Illuminate\Support\Carbon;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Tests\Concerns\SharesSupplierPdo;
|
|
|
|
uses(DatabaseTransactions::class);
|
|
uses(SharesSupplierPdo::class);
|
|
|
|
// Helper: insert a failed_webhook_jobs row
|
|
function makeFailedWebhookJob(string $exception, ?DateTimeInterface $at = null): void
|
|
{
|
|
DB::table('failed_webhook_jobs')->insert([
|
|
'failed_at' => $at ?? now(),
|
|
'exception' => $exception,
|
|
'raw_payload' => '{}',
|
|
'retry_count' => 0,
|
|
]);
|
|
}
|
|
|
|
// Helper: ensure at least one saas_admin_users row exists for FK
|
|
function ensureSystemAdmin(): int
|
|
{
|
|
$id = DB::table('saas_admin_users')->value('id');
|
|
if ($id !== null) {
|
|
return (int) $id;
|
|
}
|
|
|
|
return (int) DB::table('saas_admin_users')->insertGetId([
|
|
'email' => 'system-cron@liderra.ru',
|
|
'full_name' => 'System Cron',
|
|
'password_hash' => '$2y$12$placeholder',
|
|
'role' => 'dev_oncall',
|
|
'is_active' => true,
|
|
'created_at' => now(),
|
|
]);
|
|
}
|
|
|
|
beforeEach(function () {
|
|
ensureSystemAdmin();
|
|
});
|
|
|
|
test('does not create incident when failures are below threshold', function () {
|
|
$now = Carbon::now();
|
|
// Insert 5 failures — below default threshold of 200
|
|
for ($i = 0; $i < 5; $i++) {
|
|
makeFailedWebhookJob('App\\Exceptions\\WebhookException: connection refused', $now);
|
|
}
|
|
|
|
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
|
|
|
expect(DB::table('incidents_log')->count())->toBe(0);
|
|
});
|
|
|
|
test('creates incident when failures exceed threshold', function () {
|
|
$now = Carbon::now();
|
|
// Insert 201 failures with same exception signature
|
|
for ($i = 0; $i < 201; $i++) {
|
|
makeFailedWebhookJob('App\\Exceptions\\WebhookException: connection refused', $now);
|
|
}
|
|
|
|
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
|
|
|
expect(DB::table('incidents_log')->count())->toBe(1);
|
|
|
|
$incident = DB::table('incidents_log')->first();
|
|
expect($incident->type)->toBe('other');
|
|
expect($incident->severity)->toBe('high');
|
|
expect($incident->summary)->toContain('201');
|
|
});
|
|
|
|
test('deduplicates: does not create second incident for same ongoing storm', function () {
|
|
$now = Carbon::now();
|
|
for ($i = 0; $i < 201; $i++) {
|
|
makeFailedWebhookJob('App\\Exceptions\\WebhookException: timeout', $now);
|
|
}
|
|
|
|
// First run creates incident
|
|
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
|
expect(DB::table('incidents_log')->count())->toBe(1);
|
|
|
|
// Second run (storm still ongoing) should NOT create another incident
|
|
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
|
expect(DB::table('incidents_log')->count())->toBe(1);
|
|
});
|
|
|
|
test('creates separate incidents for different exception signatures', function () {
|
|
$now = Carbon::now();
|
|
// 201 failures with exception A
|
|
for ($i = 0; $i < 201; $i++) {
|
|
makeFailedWebhookJob('App\\Exceptions\\WebhookException: connection refused', $now);
|
|
}
|
|
// 201 failures with exception B
|
|
for ($i = 0; $i < 201; $i++) {
|
|
makeFailedWebhookJob('App\\Exceptions\\CurlException: SSL handshake failed', $now);
|
|
}
|
|
|
|
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
|
|
|
expect(DB::table('incidents_log')->count())->toBe(2);
|
|
});
|