Files
portal/app/tests/Feature/Scheduler/SchedulerHeartbeatTest.php
T
Дмитрий c76038d076 feat(ops): scheduler heartbeat — пульс 11 cron-задач + watcher (hole #6)
Закрывает дыру #6 из аудита журналирования 23.05.2026.

Что:
* `scheduler_heartbeats` таблица (SaaS-level, PK=command_name, без RLS)
* `SchedulerHeartbeatTracker` сервис — UPSERT через pgsql_supplier (BYPASSRLS),
  recordRun(callable) + recordRunResult(name, success, error, ms)
* `routes/console.php` — 11 cron-задач обёрнуты onSuccess/onFailure хуками
  (минимально-инвазивно, без правки самих джобов)
* `scheduler:check-heartbeats` команда — hourly МСК:
  - алертит при пропавшем пульсе (>2× ожидаемого интервала)
  - алертит при consecutive_failures >= 3
  - dedup 60 мин, пишет incidents_log (severity=high) + Mail на kdv1@bk.ru
* `SchedulerHeartbeatMissingMail` mailable + blade

NB: используется `onSuccess()` а не `after()` — `after()` срабатывает при любом
исходе и ложно обновлял бы last_success_at при failure (правильный поведенческий
паттерн = onSuccess + onFailure). consecutive_failures корректно растёт через
ON CONFLICT DO UPDATE +1.

Schema bump v8.29→v8.30. +1 слово в cspell-words.txt (FQCN).

Тесты: 8/8 passed (24 assertions, ~1.6s) — recordRun success/failure,
SchedulerCheckHeartbeats missing pulse + failure spike + dedup + Mailable.

Plan: docs/superpowers/plans/2026-05-23-7-holes-overview.md (#6).
2026-05-23 11:48:20 +03:00

257 lines
8.9 KiB
PHP
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<?php
declare(strict_types=1);
use App\Console\Commands\SchedulerCheckHeartbeats;
use App\Mail\SchedulerHeartbeatMissingMail;
use App\Services\SchedulerHeartbeatTracker;
use Carbon\Carbon;
use Illuminate\Foundation\Testing\DatabaseTransactions;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Mail;
use Tests\Concerns\SharesSupplierPdo;
uses(DatabaseTransactions::class, SharesSupplierPdo::class);
// Гарантируем PDO-sharing перед каждым тестом, затрагивающим pgsql_supplier.
// SharesSupplierPdo::setUpSharesSupplierPdo() вызывается автоматически через
// setUp{TraitName}, но явный beforeEach страхует от edge-cases.
beforeEach(function (): void {
DB::connection('pgsql_supplier')->setPdo(
DB::connection('pgsql')->getPdo()
);
DB::connection('pgsql_supplier')->setReadPdo(
DB::connection('pgsql')->getReadPdo()
);
});
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/**
* Получить строку heartbeat через default connection.
* В test-env оба pgsql + pgsql_supplier указывают на liderra_testing.
*/
function getHeartbeat(string $name): ?object
{
return DB::table('scheduler_heartbeats')
->where('command_name', $name)
->first();
}
function insertHeartbeat(array $data): void
{
$defaults = [
'last_run_at' => null,
'last_success_at' => null,
'last_error' => null,
'runtime_ms' => null,
'consecutive_failures' => 0,
'created_at' => now(),
'updated_at' => now(),
];
DB::table('scheduler_heartbeats')->insert(array_merge($defaults, $data));
}
/**
* Гарантирует наличие активного saas_admin_user для FK incidents_log.
* Паттерн из IncidentsWatchFailuresTest::ensureSystemAdmin().
*/
function ensureHeartbeatAdmin(): int
{
$id = DB::table('saas_admin_users')->where('is_active', true)->whereNull('deleted_at')->value('id');
if ($id !== null) {
return (int) $id;
}
return (int) DB::table('saas_admin_users')->insertGetId([
'email' => 'hb-check-admin@liderra.ru',
'full_name' => 'Heartbeat Check Admin',
'password_hash' => '$2y$12$placeholder',
'is_active' => true,
'role' => 'support',
'created_at' => now(),
]);
}
// ---------------------------------------------------------------------------
// SchedulerHeartbeatTracker::recordRun — успешный запуск
// ---------------------------------------------------------------------------
it('recordRun обновляет last_run_at и last_success_at при успехе', function (): void {
$tracker = app(SchedulerHeartbeatTracker::class);
$before = now()->subSecond();
$tracker->recordRun('test:success', fn () => null);
$row = getHeartbeat('test:success');
expect($row)->not->toBeNull('строка heartbeat не создана')
->and(Carbon::parse($row->last_run_at))->toBeGreaterThan($before)
->and(Carbon::parse($row->last_success_at))->toBeGreaterThan($before)
->and($row->consecutive_failures)->toBe(0)
->and($row->last_error)->toBeNull();
});
it('recordRun сбрасывает consecutive_failures до 0 после успеха', function (): void {
// Создаём строку с ненулевыми consecutive_failures
insertHeartbeat([
'command_name' => 'test:reset-failures',
'consecutive_failures' => 5,
'last_error' => 'prev error',
]);
$tracker = app(SchedulerHeartbeatTracker::class);
$tracker->recordRun('test:reset-failures', fn () => null);
$row = getHeartbeat('test:reset-failures');
expect($row->consecutive_failures)->toBe(0)
->and($row->last_error)->toBeNull();
});
// ---------------------------------------------------------------------------
// SchedulerHeartbeatTracker::recordRun — исключение
// ---------------------------------------------------------------------------
it('recordRun обновляет last_error и инкрементирует consecutive_failures при exception', function (): void {
$tracker = app(SchedulerHeartbeatTracker::class);
$before = now()->subSecond();
$thrown = false;
try {
$tracker->recordRun('test:fail', function (): never {
throw new RuntimeException('test error message');
});
} catch (RuntimeException) {
$thrown = true;
}
expect($thrown)->toBeTrue('исключение должно пробрасываться');
$row = getHeartbeat('test:fail');
expect($row)->not->toBeNull('строка heartbeat не создана')
->and(Carbon::parse($row->last_run_at))->toBeGreaterThan($before)
->and($row->last_success_at)->toBeNull()
->and($row->last_error)->toContain('test error message')
->and($row->consecutive_failures)->toBe(1);
});
it('recordRun инкрементирует consecutive_failures накопительно', function (): void {
insertHeartbeat([
'command_name' => 'test:multi-fail',
'consecutive_failures' => 2,
]);
$tracker = app(SchedulerHeartbeatTracker::class);
try {
$tracker->recordRun('test:multi-fail', function (): never {
throw new RuntimeException('again');
});
} catch (RuntimeException) {
}
$row = getHeartbeat('test:multi-fail');
expect($row->consecutive_failures)->toBe(3);
});
// ---------------------------------------------------------------------------
// SchedulerCheckHeartbeats — детекция пропавшего пульса
// ---------------------------------------------------------------------------
it('SchedulerCheckHeartbeats флагует команду с last_run_at старше 2× интервала', function (): void {
Mail::fake();
ensureHeartbeatAdmin();
// Команда incidents:watch-failures: интервал 10 минут → старше 20 мин = флаг
insertHeartbeat([
'command_name' => 'incidents:watch-failures',
'last_run_at' => now()->subMinutes(25),
'last_success_at' => now()->subMinutes(25),
]);
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk()->run();
$incident = DB::table('incidents_log')
->where('summary', 'like', '%incidents:watch-failures%')
->first();
expect($incident)->not->toBeNull('incident не создан для пропавшего пульса')
->and($incident->severity)->toBe('high');
});
it('SchedulerCheckHeartbeats флагует команду с consecutive_failures >= 3', function (): void {
Mail::fake();
ensureHeartbeatAdmin();
insertHeartbeat([
'command_name' => 'supplier:retry-failed',
'last_run_at' => now()->subMinutes(5),
'last_success_at' => now()->subMinutes(65),
'consecutive_failures' => 3,
]);
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk();
$incident = DB::table('incidents_log')
->where('summary', 'like', '%supplier:retry-failed%')
->first();
expect($incident)->not->toBeNull('incident не создан при consecutive_failures=3');
});
// ---------------------------------------------------------------------------
// Dedup — повторный запуск не дублирует инцидент
// ---------------------------------------------------------------------------
it('SchedulerCheckHeartbeats не дублирует incident при повторном запуске', function (): void {
Mail::fake();
ensureHeartbeatAdmin();
insertHeartbeat([
'command_name' => 'audit:verify-chains',
'last_run_at' => now()->subDays(3),
'last_success_at' => now()->subDays(3),
]);
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk();
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk();
$count = DB::table('incidents_log')
->where('summary', 'like', '%audit:verify-chains%')
->count();
expect($count)->toBe(1, 'инцидент задублирован — dedup не работает');
});
// ---------------------------------------------------------------------------
// Mailable отправляется
// ---------------------------------------------------------------------------
it('SchedulerCheckHeartbeats отправляет SchedulerHeartbeatMissingMail', function (): void {
Mail::fake();
ensureHeartbeatAdmin();
insertHeartbeat([
'command_name' => 'partitions:create-months',
'last_run_at' => now()->subHours(50),
'last_success_at' => now()->subHours(50),
]);
$this->artisan(SchedulerCheckHeartbeats::class)->assertOk();
Mail::assertSent(SchedulerHeartbeatMissingMail::class, function ($mail) {
return $mail->hasTo('kdv1@bk.ru');
});
});