257 lines
8.9 KiB
PHP
257 lines
8.9 KiB
PHP
|
|
<?php
|
|||
|
|
|
|||
|
|
declare(strict_types=1);
|
|||
|
|
|
|||
|
|
use App\Console\Commands\SchedulerCheckHeartbeats;
|
|||
|
|
use App\Mail\SchedulerHeartbeatMissingMail;
|
|||
|
|
use App\Services\SchedulerHeartbeatTracker;
|
|||
|
|
use Carbon\Carbon;
|
|||
|
|
use Illuminate\Foundation\Testing\DatabaseTransactions;
|
|||
|
|
use Illuminate\Support\Facades\DB;
|
|||
|
|
use Illuminate\Support\Facades\Mail;
|
|||
|
|
use Tests\Concerns\SharesSupplierPdo;
|
|||
|
|
|
|||
|
|
uses(DatabaseTransactions::class, SharesSupplierPdo::class);
|
|||
|
|
|
|||
|
|
// Гарантируем PDO-sharing перед каждым тестом, затрагивающим pgsql_supplier.
|
|||
|
|
// SharesSupplierPdo::setUpSharesSupplierPdo() вызывается автоматически через
|
|||
|
|
// setUp{TraitName}, но явный beforeEach страхует от edge-cases.
|
|||
|
|
beforeEach(function (): void {
|
|||
|
|
DB::connection('pgsql_supplier')->setPdo(
|
|||
|
|
DB::connection('pgsql')->getPdo()
|
|||
|
|
);
|
|||
|
|
DB::connection('pgsql_supplier')->setReadPdo(
|
|||
|
|
DB::connection('pgsql')->getReadPdo()
|
|||
|
|
);
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
// Helpers
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Получить строку heartbeat через default connection.
|
|||
|
|
* В test-env оба pgsql + pgsql_supplier указывают на liderra_testing.
|
|||
|
|
*/
|
|||
|
|
function getHeartbeat(string $name): ?object
|
|||
|
|
{
|
|||
|
|
return DB::table('scheduler_heartbeats')
|
|||
|
|
->where('command_name', $name)
|
|||
|
|
->first();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function insertHeartbeat(array $data): void
|
|||
|
|
{
|
|||
|
|
$defaults = [
|
|||
|
|
'last_run_at' => null,
|
|||
|
|
'last_success_at' => null,
|
|||
|
|
'last_error' => null,
|
|||
|
|
'runtime_ms' => null,
|
|||
|
|
'consecutive_failures' => 0,
|
|||
|
|
'created_at' => now(),
|
|||
|
|
'updated_at' => now(),
|
|||
|
|
];
|
|||
|
|
|
|||
|
|
DB::table('scheduler_heartbeats')->insert(array_merge($defaults, $data));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* Гарантирует наличие активного saas_admin_user для FK incidents_log.
|
|||
|
|
* Паттерн из IncidentsWatchFailuresTest::ensureSystemAdmin().
|
|||
|
|
*/
|
|||
|
|
function ensureHeartbeatAdmin(): int
|
|||
|
|
{
|
|||
|
|
$id = DB::table('saas_admin_users')->where('is_active', true)->whereNull('deleted_at')->value('id');
|
|||
|
|
|
|||
|
|
if ($id !== null) {
|
|||
|
|
return (int) $id;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return (int) DB::table('saas_admin_users')->insertGetId([
|
|||
|
|
'email' => 'hb-check-admin@liderra.ru',
|
|||
|
|
'full_name' => 'Heartbeat Check Admin',
|
|||
|
|
'password_hash' => '$2y$12$placeholder',
|
|||
|
|
'is_active' => true,
|
|||
|
|
'role' => 'support',
|
|||
|
|
'created_at' => now(),
|
|||
|
|
]);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
// SchedulerHeartbeatTracker::recordRun — успешный запуск
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
it('recordRun обновляет last_run_at и last_success_at при успехе', function (): void {
|
|||
|
|
$tracker = app(SchedulerHeartbeatTracker::class);
|
|||
|
|
$before = now()->subSecond();
|
|||
|
|
|
|||
|
|
$tracker->recordRun('test:success', fn () => null);
|
|||
|
|
|
|||
|
|
$row = getHeartbeat('test:success');
|
|||
|
|
|
|||
|
|
expect($row)->not->toBeNull('строка heartbeat не создана')
|
|||
|
|
->and(Carbon::parse($row->last_run_at))->toBeGreaterThan($before)
|
|||
|
|
->and(Carbon::parse($row->last_success_at))->toBeGreaterThan($before)
|
|||
|
|
->and($row->consecutive_failures)->toBe(0)
|
|||
|
|
->and($row->last_error)->toBeNull();
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
it('recordRun сбрасывает consecutive_failures до 0 после успеха', function (): void {
|
|||
|
|
// Создаём строку с ненулевыми consecutive_failures
|
|||
|
|
insertHeartbeat([
|
|||
|
|
'command_name' => 'test:reset-failures',
|
|||
|
|
'consecutive_failures' => 5,
|
|||
|
|
'last_error' => 'prev error',
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
$tracker = app(SchedulerHeartbeatTracker::class);
|
|||
|
|
$tracker->recordRun('test:reset-failures', fn () => null);
|
|||
|
|
|
|||
|
|
$row = getHeartbeat('test:reset-failures');
|
|||
|
|
|
|||
|
|
expect($row->consecutive_failures)->toBe(0)
|
|||
|
|
->and($row->last_error)->toBeNull();
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
// SchedulerHeartbeatTracker::recordRun — исключение
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
it('recordRun обновляет last_error и инкрементирует consecutive_failures при exception', function (): void {
|
|||
|
|
$tracker = app(SchedulerHeartbeatTracker::class);
|
|||
|
|
$before = now()->subSecond();
|
|||
|
|
|
|||
|
|
$thrown = false;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
$tracker->recordRun('test:fail', function (): never {
|
|||
|
|
throw new RuntimeException('test error message');
|
|||
|
|
});
|
|||
|
|
} catch (RuntimeException) {
|
|||
|
|
$thrown = true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
expect($thrown)->toBeTrue('исключение должно пробрасываться');
|
|||
|
|
|
|||
|
|
$row = getHeartbeat('test:fail');
|
|||
|
|
|
|||
|
|
expect($row)->not->toBeNull('строка heartbeat не создана')
|
|||
|
|
->and(Carbon::parse($row->last_run_at))->toBeGreaterThan($before)
|
|||
|
|
->and($row->last_success_at)->toBeNull()
|
|||
|
|
->and($row->last_error)->toContain('test error message')
|
|||
|
|
->and($row->consecutive_failures)->toBe(1);
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
it('recordRun инкрементирует consecutive_failures накопительно', function (): void {
|
|||
|
|
insertHeartbeat([
|
|||
|
|
'command_name' => 'test:multi-fail',
|
|||
|
|
'consecutive_failures' => 2,
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
$tracker = app(SchedulerHeartbeatTracker::class);
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
$tracker->recordRun('test:multi-fail', function (): never {
|
|||
|
|
throw new RuntimeException('again');
|
|||
|
|
});
|
|||
|
|
} catch (RuntimeException) {
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
$row = getHeartbeat('test:multi-fail');
|
|||
|
|
expect($row->consecutive_failures)->toBe(3);
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
// SchedulerCheckHeartbeats — детекция пропавшего пульса
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
it('SchedulerCheckHeartbeats флагует команду с last_run_at старше 2× интервала', function (): void {
|
|||
|
|
Mail::fake();
|
|||
|
|
|
|||
|
|
ensureHeartbeatAdmin();
|
|||
|
|
|
|||
|
|
// Команда incidents:watch-failures: интервал 10 минут → старше 20 мин = флаг
|
|||
|
|
insertHeartbeat([
|
|||
|
|
'command_name' => 'incidents:watch-failures',
|
|||
|
|
'last_run_at' => now()->subMinutes(25),
|
|||
|
|
'last_success_at' => now()->subMinutes(25),
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk()->run();
|
|||
|
|
|
|||
|
|
$incident = DB::table('incidents_log')
|
|||
|
|
->where('summary', 'like', '%incidents:watch-failures%')
|
|||
|
|
->first();
|
|||
|
|
|
|||
|
|
expect($incident)->not->toBeNull('incident не создан для пропавшего пульса')
|
|||
|
|
->and($incident->severity)->toBe('high');
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
it('SchedulerCheckHeartbeats флагует команду с consecutive_failures >= 3', function (): void {
|
|||
|
|
Mail::fake();
|
|||
|
|
|
|||
|
|
ensureHeartbeatAdmin();
|
|||
|
|
|
|||
|
|
insertHeartbeat([
|
|||
|
|
'command_name' => 'supplier:retry-failed',
|
|||
|
|
'last_run_at' => now()->subMinutes(5),
|
|||
|
|
'last_success_at' => now()->subMinutes(65),
|
|||
|
|
'consecutive_failures' => 3,
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk();
|
|||
|
|
|
|||
|
|
$incident = DB::table('incidents_log')
|
|||
|
|
->where('summary', 'like', '%supplier:retry-failed%')
|
|||
|
|
->first();
|
|||
|
|
|
|||
|
|
expect($incident)->not->toBeNull('incident не создан при consecutive_failures=3');
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
// Dedup — повторный запуск не дублирует инцидент
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
it('SchedulerCheckHeartbeats не дублирует incident при повторном запуске', function (): void {
|
|||
|
|
Mail::fake();
|
|||
|
|
|
|||
|
|
ensureHeartbeatAdmin();
|
|||
|
|
|
|||
|
|
insertHeartbeat([
|
|||
|
|
'command_name' => 'audit:verify-chains',
|
|||
|
|
'last_run_at' => now()->subDays(3),
|
|||
|
|
'last_success_at' => now()->subDays(3),
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk();
|
|||
|
|
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk();
|
|||
|
|
|
|||
|
|
$count = DB::table('incidents_log')
|
|||
|
|
->where('summary', 'like', '%audit:verify-chains%')
|
|||
|
|
->count();
|
|||
|
|
|
|||
|
|
expect($count)->toBe(1, 'инцидент задублирован — dedup не работает');
|
|||
|
|
});
|
|||
|
|
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
// Mailable отправляется
|
|||
|
|
// ---------------------------------------------------------------------------
|
|||
|
|
|
|||
|
|
it('SchedulerCheckHeartbeats отправляет SchedulerHeartbeatMissingMail', function (): void {
|
|||
|
|
Mail::fake();
|
|||
|
|
|
|||
|
|
ensureHeartbeatAdmin();
|
|||
|
|
|
|||
|
|
insertHeartbeat([
|
|||
|
|
'command_name' => 'partitions:create-months',
|
|||
|
|
'last_run_at' => now()->subHours(50),
|
|||
|
|
'last_success_at' => now()->subHours(50),
|
|||
|
|
]);
|
|||
|
|
|
|||
|
|
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk();
|
|||
|
|
|
|||
|
|
Mail::assertSent(SchedulerHeartbeatMissingMail::class, function ($mail) {
|
|||
|
|
return $mail->hasTo('kdv1@bk.ru');
|
|||
|
|
});
|
|||
|
|
});
|