Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dfa14862b4 | |||
| 9799c0eb0d | |||
| 316bc2d852 |
@@ -33,7 +33,8 @@ class IncidentsWatchFailures extends Command
|
||||
{--threshold-daily=50 : Порог суммы за 24ч для failed_jobs}
|
||||
{--persistent-hours=3 : Порог возраста persistent-exception для failed_jobs}
|
||||
{--dedup-window=60 : Окно дедупа открытых инцидентов в минутах}
|
||||
{--threshold-single-lead=1000 : Порог storm detection: failures одного supplier_lead_id за окно}';
|
||||
{--threshold-single-lead=1000 : Порог storm detection: failures одного supplier_lead_id за окно}
|
||||
{--threshold-manual-queue=10 : Порог spike ручной очереди поставщика (за окно) → возможное падение кабинета}';
|
||||
|
||||
protected $description = 'Сканирует failed_webhook_jobs и failed_jobs, создаёт incidents_log на превышение порогов';
|
||||
|
||||
@@ -47,6 +48,7 @@ class IncidentsWatchFailures extends Command
|
||||
$dedupMinutes = (int) $this->option('dedup-window');
|
||||
|
||||
$thresholdSingleLead = (int) $this->option('threshold-single-lead');
|
||||
$thresholdManualQueue = (int) $this->option('threshold-manual-queue');
|
||||
|
||||
$since = Carbon::now()->subMinutes($windowMinutes);
|
||||
$since24h = Carbon::now()->subHours(24);
|
||||
@@ -221,6 +223,35 @@ class IncidentsWatchFailures extends Command
|
||||
}
|
||||
}
|
||||
|
||||
// ===== БЛОК 6: supplier-manual-queue spike =====
|
||||
// Пачка проектов, ушедших в ручную очередь за короткое окно = вероятное падение
|
||||
// кабинета поставщика. Инцидент дедупим (60 мин, чтобы не засорять журнал), НО письмо
|
||||
// шлём на КАЖДОМ прогоне пока спайк держится — «сигналит как ненормальный» при реальной
|
||||
// аварии, одним сводным письмом на оба адреса (не N писем на проект).
|
||||
if ($thresholdManualQueue > 0) {
|
||||
$manualQueueCount = (int) DB::connection(self::DB_CONNECTION)
|
||||
->table('supplier_manual_sync_queue')
|
||||
->where('created_at', '>=', $since)
|
||||
->count();
|
||||
|
||||
if ($manualQueueCount >= $thresholdManualQueue) {
|
||||
$dedupKey = 'supplier-manual-queue-spike';
|
||||
$summary = "Похоже, кабинет поставщика упал: {$manualQueueCount} проект(ов) ушло в ручную очередь за {$windowMinutes} мин.";
|
||||
|
||||
// Инцидент — только если открытого с той же сигнатурой нет (дедуп 60 мин).
|
||||
if (! $this->isDup($dedupKey, $dedupAt)) {
|
||||
$this->createIncident($adminId, 'other', 'high', $summary, $since, $now, $dedupKey, sendMail: false);
|
||||
$created++;
|
||||
}
|
||||
|
||||
// Письмо — каждый прогон при активном спайке, на оба адреса.
|
||||
Mail::to(['kdv1@bk.ru', 'ops@liderra.ru'])
|
||||
->send(new IncidentDetectedMail($summary, 'high'));
|
||||
|
||||
$this->info("Supplier manual-queue spike [high]: {$manualQueueCount} in {$windowMinutes}m");
|
||||
}
|
||||
}
|
||||
|
||||
$this->info("Done. Created {$created} incident(s).");
|
||||
|
||||
return self::SUCCESS;
|
||||
@@ -245,6 +276,7 @@ class IncidentsWatchFailures extends Command
|
||||
Carbon $startedAt,
|
||||
Carbon $now,
|
||||
string $dedupKey = '',
|
||||
bool $sendMail = true,
|
||||
): void {
|
||||
DB::connection(self::DB_CONNECTION)->table('incidents_log')->insert([
|
||||
'type' => $type,
|
||||
@@ -259,7 +291,7 @@ class IncidentsWatchFailures extends Command
|
||||
'updated_at' => $now,
|
||||
]);
|
||||
|
||||
if ($severity === 'high') {
|
||||
if ($sendMail && $severity === 'high') {
|
||||
Mail::to('kdv1@bk.ru')->send(new IncidentDetectedMail($summary, $severity));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,6 +134,10 @@ class SyncSupplierProjectsJob implements ShouldQueue
|
||||
$groups[$key]['projects'][] = $project;
|
||||
}
|
||||
|
||||
// Активные сегодня источники (по слепку) — их заказы НЕ гасим в deactivation-проходе.
|
||||
/** @var array<string, bool> $activeKeys */
|
||||
$activeKeys = array_fill_keys(array_keys($groups), true);
|
||||
|
||||
// 3. Sync each group
|
||||
try {
|
||||
foreach ($groups as $group) {
|
||||
@@ -192,6 +196,13 @@ class SyncSupplierProjectsJob implements ShouldQueue
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Deactivation (money-critical, 18:00 ДО снимка поставщика в 21:00): гасим
|
||||
// заказы источников, у которых на сегодняшний слепок не осталось ни одного
|
||||
// активного проекта. Пропускаем при aborted (time-budget/mass-fail/auth).
|
||||
if (! $aborted) {
|
||||
$this->deactivateOrphanedOrders($activeKeys);
|
||||
}
|
||||
} finally {
|
||||
$this->recordRunSummary(
|
||||
startedAt: $startedAt,
|
||||
@@ -205,6 +216,103 @@ class SyncSupplierProjectsJob implements ShouldQueue
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deactivation (18:00, money-critical — owner-reported 2026-07-08, verified on prod live DB):
|
||||
* гасим у поставщика заказы источников, у которых на СЕГОДНЯШНИЙ слепок не осталось ни
|
||||
* одного активного проекта. Поставщик фиксирует завтрашнюю заливку своим снимком в 21:00,
|
||||
* поэтому единственное окно остановить заказ — этот 18:00-прогон ДО снимка. Без этого
|
||||
* осиротевший заказ оставался включённым (галочка вкл, лимит > 0) → поставщик лил лиды и
|
||||
* списывал за остановленный клиентом проект.
|
||||
*
|
||||
* Активность — из СЛЕПКА ($activeKeys = ключи sync-групп за завтра), НЕ из live is_active:
|
||||
* слепок-инвариант — проект, paused'нутый уже ПОСЛЕ снятия слепка, честно доезжает
|
||||
* поставщику (см. collectEligibleProjects); гасим только то, чего в слепке нет.
|
||||
*
|
||||
* Осиротевший = supplier_project с external_id, inactive_since IS NULL, чей
|
||||
* (signal_type|unique_key) не входит в $activeKeys. Шлём updateProject(status='paused') с
|
||||
* ненулевым лимитом (кабинет отклоняет limit=0 даже при паузе), помечаем inactive_since
|
||||
* ТОЛЬКО после успешной отправки (иначе — ретрай следующим прогоном). Реактивацию при
|
||||
* возврате проекта делают CleanupInactiveSupplierProjectsJob (Phase A) + syncGroup(active).
|
||||
*
|
||||
* @param array<string, bool> $activeKeys ключи «signal_type|identifier» активных групп
|
||||
*/
|
||||
private function deactivateOrphanedOrders(array $activeKeys): int
|
||||
{
|
||||
$paused = 0;
|
||||
|
||||
// get() (не cursor) — на shared PDO тестов курсор конфликтует с write'ами в цикле;
|
||||
// набор ограничен (активные + ещё-не-погашенные осиротевшие заказы).
|
||||
$candidates = SupplierProject::on(self::DB_CONNECTION)
|
||||
->whereNotNull('supplier_external_id')
|
||||
->whereNull('inactive_since')
|
||||
->orderBy('id')
|
||||
->get();
|
||||
|
||||
foreach ($candidates as $sp) {
|
||||
if (isset($activeKeys[$sp->signal_type.'|'.$sp->unique_key])) {
|
||||
continue; // источник ещё активен в сегодняшнем слепке — не трогаем
|
||||
}
|
||||
if (now()->timezone('Europe/Moscow')->format('H:i') >= self::TIME_BUDGET_CUTOFF) {
|
||||
break; // время вышло — недогашенное подберёт следующий прогон
|
||||
}
|
||||
|
||||
$regions = array_map('intval', (array) ($sp->current_regions ?? []));
|
||||
$workdays = $sp->current_workdays !== null && $sp->current_workdays !== []
|
||||
? array_map('intval', (array) $sp->current_workdays)
|
||||
: [1, 2, 3, 4, 5, 6, 7];
|
||||
$tag = count($regions) === 1
|
||||
? (RussianRegions::CODE_TO_NAME[$regions[0]] ?? (string) $regions[0])
|
||||
: 'РФ';
|
||||
|
||||
$dto = new SupplierProjectDto(
|
||||
platform: $sp->platform,
|
||||
signalType: (string) $sp->signal_type,
|
||||
uniqueKey: (string) $sp->unique_key,
|
||||
limit: max(1, (int) $sp->current_limit),
|
||||
workdays: $workdays,
|
||||
regions: $regions,
|
||||
regionsReverse: false,
|
||||
status: 'paused',
|
||||
tag: $tag,
|
||||
platforms: [$sp->platform],
|
||||
);
|
||||
|
||||
try {
|
||||
$this->channel->updateProject((int) $sp->supplier_external_id, $dto);
|
||||
} catch (Throwable $e) {
|
||||
// Transient/portal error — inactive_since НЕ ставим, ретрай следующим прогоном.
|
||||
Log::warning('supplier.sync.deactivate_failed', [
|
||||
'unique_key' => (string) $sp->unique_key,
|
||||
'platform' => $sp->platform,
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$sp->forceFill([
|
||||
'sync_status' => 'ok',
|
||||
'last_synced_at' => now(),
|
||||
'inactive_since' => now(),
|
||||
])->save();
|
||||
|
||||
SupplierSyncLog::on(self::DB_CONNECTION)->create([
|
||||
'supplier_project_id' => $sp->id,
|
||||
'action' => 'update',
|
||||
'http_status' => 200,
|
||||
'created_at' => now(),
|
||||
]);
|
||||
|
||||
$paused++;
|
||||
}
|
||||
|
||||
if ($paused > 0) {
|
||||
Log::info('supplier.sync.deactivated_orphaned', ['count' => $paused]);
|
||||
}
|
||||
|
||||
return $paused;
|
||||
}
|
||||
|
||||
/**
|
||||
* Эпик 5: одна строка-сводка в supplier_sync_runs по завершении заливки.
|
||||
* Пишется и при раннем abort (time-budget / mass-fail / auth) — через finally.
|
||||
|
||||
@@ -21,7 +21,6 @@ use App\Services\Supplier\ProcessFactory;
|
||||
use App\Services\Supplier\SymfonyProcessFactory;
|
||||
use Illuminate\Auth\Notifications\ResetPassword;
|
||||
use Illuminate\Cache\RateLimiting\Limit;
|
||||
use Illuminate\Contracts\Mail\Mailer;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Support\Facades\Auth;
|
||||
use Illuminate\Support\Facades\Hash;
|
||||
@@ -49,7 +48,6 @@ class AppServiceProvider extends ServiceProvider
|
||||
fn ($app) => new FailoverProjectChannel(
|
||||
$app->make(AjaxProjectChannel::class),
|
||||
$app->make(FormProjectChannel::class),
|
||||
$app->make(Mailer::class),
|
||||
),
|
||||
);
|
||||
|
||||
|
||||
@@ -7,13 +7,11 @@ namespace App\Services\Supplier\Channel;
|
||||
use App\Exceptions\Supplier\SupplierAuthException;
|
||||
use App\Exceptions\Supplier\SupplierClientException;
|
||||
use App\Exceptions\Supplier\SupplierTransientException;
|
||||
use App\Mail\SupplierCriticalAlertMail;
|
||||
use App\Models\Project;
|
||||
use App\Models\SupplierManualSyncQueue;
|
||||
use App\Services\Supplier\Channel\Exceptions\TierEscalatedException;
|
||||
use App\Services\Supplier\Channel\Exceptions\WindowDeferredException;
|
||||
use App\Services\Supplier\Dto\SupplierProjectDto;
|
||||
use Illuminate\Contracts\Mail\Mailer;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Throwable;
|
||||
|
||||
@@ -31,7 +29,6 @@ final class FailoverProjectChannel implements SupplierProjectChannel
|
||||
public function __construct(
|
||||
private readonly SupplierProjectChannel $tier1,
|
||||
private readonly SupplierProjectChannel $tier2,
|
||||
private readonly Mailer $mailer,
|
||||
) {}
|
||||
|
||||
public function createProject(SupplierProjectDto $dto): int
|
||||
@@ -81,7 +78,6 @@ final class FailoverProjectChannel implements SupplierProjectChannel
|
||||
} catch (SupplierClientException|SupplierAuthException $e) {
|
||||
try {
|
||||
$id = $this->tier2->createProject($dto);
|
||||
$this->alertFailoverToForm($project, 'create', $e);
|
||||
|
||||
return $id;
|
||||
} catch (Throwable $tier2Error) {
|
||||
@@ -108,7 +104,6 @@ final class FailoverProjectChannel implements SupplierProjectChannel
|
||||
} catch (SupplierClientException|SupplierAuthException $e) {
|
||||
try {
|
||||
$this->tier2->updateProject($externalId, $dto);
|
||||
$this->alertFailoverToForm($project, 'update', $e);
|
||||
|
||||
return;
|
||||
} catch (Throwable $tier2Error) {
|
||||
@@ -147,24 +142,9 @@ final class FailoverProjectChannel implements SupplierProjectChannel
|
||||
'created_at' => now(),
|
||||
]);
|
||||
|
||||
$this->mailer->to((string) config('services.supplier.alert_email'))
|
||||
->queue(new SupplierCriticalAlertMail(
|
||||
alertType: 'manual_required',
|
||||
details: "Project #{$project->id} ({$dto->platform}/{$dto->uniqueKey}) — {$operation} queued #{$row->id}, reason: {$reason}. Cause: ".mb_substr($cause->getMessage(), 0, 300),
|
||||
));
|
||||
|
||||
throw new TierEscalatedException($row->id, $reason);
|
||||
}
|
||||
|
||||
private function alertFailoverToForm(Project $project, string $operation, Throwable $cause): void
|
||||
{
|
||||
$this->mailer->to((string) config('services.supplier.alert_email'))
|
||||
->queue(new SupplierCriticalAlertMail(
|
||||
alertType: 'failover_to_form',
|
||||
details: "Project #{$project->id} {$operation}: Tier 1 (AJAX) failed, Tier 2 (browser) succeeded. Cause: ".mb_substr($cause->getMessage(), 0, 300),
|
||||
));
|
||||
}
|
||||
|
||||
/**
|
||||
* Портальная сверка: ищет уже существующий проект на портале по тройке
|
||||
* (platform, signal_type, unique_key). Возвращает external_id найденного
|
||||
|
||||
@@ -1317,7 +1317,7 @@ parameters:
|
||||
-
|
||||
message: '#^Call to an undefined method Pest\\PendingCalls\\TestCall\:\:artisan\(\)\.$#'
|
||||
identifier: method.notFound
|
||||
count: 5
|
||||
count: 9
|
||||
path: tests/Feature/Console/IncidentsWatchFailuresTest.php
|
||||
|
||||
-
|
||||
|
||||
@@ -2,9 +2,13 @@
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
use App\Mail\IncidentDetectedMail;
|
||||
use App\Models\Project;
|
||||
use App\Models\Tenant;
|
||||
use Illuminate\Foundation\Testing\DatabaseTransactions;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Mail;
|
||||
use Tests\Concerns\SharesSupplierPdo;
|
||||
|
||||
uses(DatabaseTransactions::class);
|
||||
@@ -107,3 +111,63 @@ test('creates separate incidents for different exception signatures', function (
|
||||
|
||||
expect(DB::table('incidents_log')->count())->toBe(2);
|
||||
});
|
||||
|
||||
// ===== Блок 6: supplier-manual-queue spike =====
|
||||
|
||||
// Helper: seed a project + N supplier_manual_sync_queue rows within the window
|
||||
function seedManualQueue(int $n, ?DateTimeInterface $at = null): int
|
||||
{
|
||||
$project = Project::factory()->for(Tenant::factory())->create();
|
||||
for ($i = 0; $i < $n; $i++) {
|
||||
DB::table('supplier_manual_sync_queue')->insert([
|
||||
'project_id' => $project->id,
|
||||
'platform' => 'B1',
|
||||
'operation' => 'create',
|
||||
'external_id' => null,
|
||||
'payload_snapshot' => '{}',
|
||||
'failure_reason' => 'portal_unreachable',
|
||||
'status' => 'pending',
|
||||
'created_at' => $at ?? now(),
|
||||
]);
|
||||
}
|
||||
|
||||
return $project->id;
|
||||
}
|
||||
|
||||
test('manual-queue spike below threshold: no incident, no mail', function () {
|
||||
Mail::fake();
|
||||
seedManualQueue(3, now()); // < default threshold 10
|
||||
|
||||
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
||||
|
||||
expect(DB::table('incidents_log')->where('root_cause', 'supplier-manual-queue-spike')->count())->toBe(0);
|
||||
Mail::assertNothingSent();
|
||||
});
|
||||
|
||||
test('manual-queue spike at/above threshold: high incident + mail to both addresses', function () {
|
||||
Mail::fake();
|
||||
seedManualQueue(10, now()); // == default threshold 10
|
||||
|
||||
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
||||
|
||||
$incident = DB::table('incidents_log')->where('root_cause', 'supplier-manual-queue-spike')->first();
|
||||
expect($incident)->not->toBeNull();
|
||||
expect($incident->severity)->toBe('high');
|
||||
expect($incident->summary)->toContain('10');
|
||||
|
||||
Mail::assertSent(IncidentDetectedMail::class, function ($mail) {
|
||||
return $mail->hasTo('kdv1@bk.ru') && $mail->hasTo('ops@liderra.ru');
|
||||
});
|
||||
});
|
||||
|
||||
test('manual-queue spike dedup: second run in window makes no new incident BUT re-sends mail', function () {
|
||||
Mail::fake();
|
||||
seedManualQueue(12, now());
|
||||
|
||||
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
||||
$this->artisan('incidents:watch-failures')->assertSuccessful();
|
||||
|
||||
// Инцидент один (дедуп 60 мин), а письмо ушло на каждом прогоне (громкость).
|
||||
expect(DB::table('incidents_log')->where('root_cause', 'supplier-manual-queue-spike')->count())->toBe(1);
|
||||
Mail::assertSent(IncidentDetectedMail::class, 2);
|
||||
});
|
||||
|
||||
@@ -5,7 +5,6 @@ declare(strict_types=1);
|
||||
use App\Exceptions\Supplier\SupplierAuthException;
|
||||
use App\Exceptions\Supplier\SupplierClientException;
|
||||
use App\Exceptions\Supplier\SupplierTransientException;
|
||||
use App\Mail\SupplierCriticalAlertMail;
|
||||
use App\Models\Project;
|
||||
use App\Models\SupplierManualSyncQueue;
|
||||
use App\Models\Tenant;
|
||||
@@ -14,7 +13,6 @@ use App\Services\Supplier\Channel\Exceptions\WindowDeferredException;
|
||||
use App\Services\Supplier\Channel\FailoverProjectChannel;
|
||||
use App\Services\Supplier\Channel\SupplierProjectChannel;
|
||||
use App\Services\Supplier\Dto\SupplierProjectDto;
|
||||
use Illuminate\Contracts\Mail\Mailer;
|
||||
use Illuminate\Foundation\Testing\DatabaseTransactions;
|
||||
use Illuminate\Support\Facades\Mail;
|
||||
|
||||
@@ -46,7 +44,6 @@ function makeFailover(SupplierProjectChannel $tier1, ?SupplierProjectChannel $ti
|
||||
return [];
|
||||
}
|
||||
},
|
||||
app(Mailer::class),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -125,7 +122,7 @@ it('createProject — Tier 1 transient-exhausted: skips Tier 2, jumps to Tier 3
|
||||
|
||||
expect($tier2Called)->toBeFalse();
|
||||
expect(SupplierManualSyncQueue::where('project_id', $project->id)->where('failure_reason', 'portal_unreachable')->count())->toBe(1);
|
||||
Mail::assertQueued(SupplierCriticalAlertMail::class);
|
||||
Mail::assertNothingQueued();
|
||||
});
|
||||
|
||||
it('createProject — Tier 1 client-exc → Tier 2 success: no queue', function (): void {
|
||||
@@ -165,7 +162,7 @@ it('createProject — Tier 1 client-exc → Tier 2 success: no queue', function
|
||||
|
||||
expect($id)->toBe(800001);
|
||||
expect(SupplierManualSyncQueue::count())->toBe(0);
|
||||
Mail::assertQueued(SupplierCriticalAlertMail::class); // failover_to_form alert
|
||||
Mail::assertNothingQueued(); // failover через форму — успех, не тревога
|
||||
});
|
||||
|
||||
it('createProject — Tier 1 client-exc + Tier 2 fail: Tier 3 queue, manual_required alert', function (): void {
|
||||
@@ -205,7 +202,7 @@ it('createProject — Tier 1 client-exc + Tier 2 fail: Tier 3 queue, manual_requ
|
||||
->toThrow(TierEscalatedException::class);
|
||||
|
||||
expect(SupplierManualSyncQueue::where('project_id', $project->id)->where('status', 'pending')->count())->toBe(1);
|
||||
Mail::assertQueued(SupplierCriticalAlertMail::class);
|
||||
Mail::assertNothingQueued();
|
||||
});
|
||||
|
||||
it('createProject — Tier 1 auth-exc → Tier 2 success', function (): void {
|
||||
|
||||
@@ -4,7 +4,6 @@ declare(strict_types=1);
|
||||
|
||||
use App\Exceptions\Supplier\SupplierClientException;
|
||||
use App\Exceptions\Supplier\SupplierTransientException;
|
||||
use App\Mail\SupplierCriticalAlertMail;
|
||||
use App\Models\Project;
|
||||
use App\Models\SupplierManualSyncQueue;
|
||||
use App\Models\Tenant;
|
||||
@@ -12,13 +11,12 @@ use App\Services\Supplier\Channel\Exceptions\TierEscalatedException;
|
||||
use App\Services\Supplier\Channel\FailoverProjectChannel;
|
||||
use App\Services\Supplier\Channel\SupplierProjectChannel;
|
||||
use App\Services\Supplier\Dto\SupplierProjectDto;
|
||||
use Illuminate\Contracts\Mail\Mailer;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Mail;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
test('Tier-1 fail + Tier-2 fail → Tier-3 escalation creates manual queue row + queues alert mail', function (): void {
|
||||
test('Tier-1 fail + Tier-2 fail → Tier-3 escalation creates manual queue row, no per-project alert mail', function (): void {
|
||||
Mail::fake();
|
||||
config(['services.supplier.alert_email' => 'ops@liderra.local']);
|
||||
|
||||
@@ -32,7 +30,7 @@ test('Tier-1 fail + Tier-2 fail → Tier-3 escalation creates manual queue row +
|
||||
$tier2 = mock(SupplierProjectChannel::class);
|
||||
$tier2->shouldReceive('createProject')->andThrow(new RuntimeException('Tier-2 manage-project.js selector break'));
|
||||
|
||||
$channel = new FailoverProjectChannel($tier1, $tier2, app(Mailer::class));
|
||||
$channel = new FailoverProjectChannel($tier1, $tier2);
|
||||
|
||||
$dto = new SupplierProjectDto(
|
||||
platform: 'B1', signalType: 'site', uniqueKey: 'failover-smoke.example',
|
||||
@@ -44,7 +42,7 @@ test('Tier-1 fail + Tier-2 fail → Tier-3 escalation creates manual queue row +
|
||||
|
||||
expect(SupplierManualSyncQueue::where('project_id', $project->id)->count())->toBe(1);
|
||||
|
||||
Mail::assertQueued(SupplierCriticalAlertMail::class, fn ($m) => $m->alertType === 'manual_required');
|
||||
Mail::assertNothingQueued(); // per-project тревога убрана — запись в ручной очереди достаточна
|
||||
});
|
||||
|
||||
test('Tier-1 transient fail (portal unreachable) bypasses Tier-2 and goes straight to Tier-3', function (): void {
|
||||
@@ -61,7 +59,7 @@ test('Tier-1 transient fail (portal unreachable) bypasses Tier-2 and goes straig
|
||||
$tier2 = mock(SupplierProjectChannel::class);
|
||||
$tier2->shouldNotReceive('createProject'); // КЛЮЧЕВОЕ — transient НЕ должен попасть в tier-2
|
||||
|
||||
$channel = new FailoverProjectChannel($tier1, $tier2, app(Mailer::class));
|
||||
$channel = new FailoverProjectChannel($tier1, $tier2);
|
||||
|
||||
$dto = new SupplierProjectDto(
|
||||
platform: 'B1', signalType: 'site', uniqueKey: 'transient-smoke.example',
|
||||
|
||||
@@ -577,3 +577,113 @@ test('nightly: re-creates donor on portal when its external_id no longer exists
|
||||
expect($sps)->toHaveCount(3);
|
||||
expect($sps->pluck('supplier_external_id')->all())->toBe(['8001', '8002', '8003']);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Deactivation (18:00): turn OFF orders whose source has no active project left.
|
||||
// Money-critical (owner-reported 2026-07-08, verified on prod live DB): prod runs
|
||||
// in BATCH mode. The supplier fixes tomorrow's delivery by its 21:00 snapshot, so the
|
||||
// ONLY window to STOP an order is the 18:00 batch sync — before that snapshot. This
|
||||
// nightly job only processed ACTIVE projects and never turned OFF a source whose
|
||||
// projects are all now paused → the paused order stayed active at the supplier → next
|
||||
// day leads (and charges) kept coming. The 18:00 job must push status=paused for such
|
||||
// orphaned orders before 21:00.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('nightly 18:00: a source with no active project left is turned OFF (status=paused) at the supplier', function (): void {
|
||||
$tenant = Tenant::factory()->create();
|
||||
$common = '79997778899';
|
||||
|
||||
// Project is PAUSED → excluded from tomorrow's snapshot (SnapshotProjectRoutingJob
|
||||
// filters is_active=true). Deliberately NO insertSnapshotForTomorrow — nothing active
|
||||
// references this source tonight, so its supplier order is orphaned.
|
||||
Project::factory()->create([
|
||||
'tenant_id' => $tenant->id, 'is_active' => false, 'signal_type' => 'call',
|
||||
'signal_identifier' => $common, 'daily_limit_target' => 6, 'delivery_days_mask' => 127, 'regions' => [],
|
||||
]);
|
||||
|
||||
// Pre-seed the supplier orders created earlier when the project was live (numeric
|
||||
// external ids so updateProject's (int) cast keeps id != 0).
|
||||
foreach (['B1' => '90001', 'B2' => '90002', 'B3' => '90003'] as $platform => $ext) {
|
||||
SupplierProject::on('pgsql_supplier')->create([
|
||||
'platform' => $platform, 'signal_type' => 'call', 'unique_key' => $common,
|
||||
'subject_code' => null, 'supplier_external_id' => $ext, 'current_limit' => 2,
|
||||
'current_workdays' => [1, 2, 3, 4, 5, 6, 7], 'current_regions' => [], 'sync_status' => 'ok',
|
||||
'last_synced_at' => now()->subDay(),
|
||||
]);
|
||||
}
|
||||
|
||||
$savePayloads = [];
|
||||
Http::fake([
|
||||
'crm.bp-gr.ru/admin/visit/rt-project-save' => function ($request) use (&$savePayloads) {
|
||||
$savePayloads[] = $request->data();
|
||||
|
||||
return Http::response(['status' => 'OK', 'message' => '', 'id' => (string) ($request->data()['id'] ?? 0)], 200);
|
||||
},
|
||||
'crm.bp-gr.ru/admin/visit/rt-projects-load*' => Http::response(['projects' => []], 200),
|
||||
]);
|
||||
|
||||
(new SyncSupplierProjectsJob)->handle(app(AjaxProjectChannel::class));
|
||||
|
||||
// The 3 orphaned orders were turned OFF: update (id != 0) with status=false.
|
||||
$offCalls = array_values(array_filter(
|
||||
$savePayloads,
|
||||
fn ($p) => (int) ($p['id'] ?? 0) !== 0 && ($p['status'] ?? null) === false
|
||||
));
|
||||
expect($offCalls)->toHaveCount(3);
|
||||
// Non-zero limit kept (portal rejects limit=0 even when paused).
|
||||
foreach ($offCalls as $p) {
|
||||
expect((int) $p['limit'])->toBeGreaterThan(0);
|
||||
}
|
||||
// Local rows marked inactive_since so cleanup/UI reflect the pause.
|
||||
expect(SupplierProject::on('pgsql_supplier')->where('unique_key', $common)->whereNotNull('inactive_since')->count())->toBe(3);
|
||||
});
|
||||
|
||||
test('nightly 18:00: a deleted project leftover order is turned OFF but KEPT in DB while its snapshot tail still routes', function (): void {
|
||||
// Composition of the new deactivation pass with delete-defer (DeleteSupplierProjectTailGuardTest):
|
||||
// when a project is deleted but its donor is deferred (so already-ordered tail leads still
|
||||
// route via the snapshot), the 18:00 pass must still turn the order OFF at the supplier
|
||||
// (stop FUTURE ordering) — WITHOUT deleting the local donor (the tail still needs it).
|
||||
$tenant = Tenant::factory()->create();
|
||||
$common = '79996665544';
|
||||
|
||||
// Donor exists (created when the project was live). Project already hard-deleted → no pivot
|
||||
// links, and NOT in tomorrow's active snapshot.
|
||||
$sp = SupplierProject::on('pgsql_supplier')->create([
|
||||
'platform' => 'B1', 'signal_type' => 'call', 'unique_key' => $common,
|
||||
'subject_code' => null, 'supplier_external_id' => '95001', 'current_limit' => 3,
|
||||
'current_workdays' => [1, 2, 3, 4, 5, 6, 7], 'current_regions' => [], 'sync_status' => 'ok',
|
||||
'last_synced_at' => now()->subDay(),
|
||||
]);
|
||||
// Tail snapshot for the source (fake project_id → not an active project, mirrors the
|
||||
// delete tail-guard fixture). LeadRouter still needs the donor for tail matching.
|
||||
DB::table('project_routing_snapshots')->insert([
|
||||
'snapshot_date' => Carbon::tomorrow('Europe/Moscow')->toDateString(),
|
||||
'project_id' => 987654, 'tenant_id' => $tenant->id, 'daily_limit' => 5,
|
||||
'delivery_days_mask' => 127, 'regions' => '{}', 'signal_type' => 'call',
|
||||
'signal_identifier' => $common, 'sms_senders' => null, 'sms_keyword' => null,
|
||||
'expected_volume' => 5, 'delivered_count' => 0, 'created_at' => now(),
|
||||
]);
|
||||
|
||||
$savePayloads = [];
|
||||
Http::fake([
|
||||
'crm.bp-gr.ru/admin/visit/rt-project-save' => function ($request) use (&$savePayloads) {
|
||||
$savePayloads[] = $request->data();
|
||||
|
||||
return Http::response(['status' => 'OK', 'message' => '', 'id' => (string) ($request->data()['id'] ?? 0)], 200);
|
||||
},
|
||||
'crm.bp-gr.ru/admin/visit/rt-projects-load*' => Http::response(['projects' => []], 200),
|
||||
]);
|
||||
|
||||
(new SyncSupplierProjectsJob)->handle(app(AjaxProjectChannel::class));
|
||||
|
||||
// Order turned OFF at the supplier (update id != 0, status=false) — stops future ordering.
|
||||
$off = array_values(array_filter(
|
||||
$savePayloads,
|
||||
fn ($p) => (int) ($p['id'] ?? 0) !== 0 && ($p['status'] ?? null) === false
|
||||
));
|
||||
expect($off)->toHaveCount(1);
|
||||
// The local donor record is KEPT (not deleted) so the snapshot tail still routes; marked inactive.
|
||||
$fresh = SupplierProject::on('pgsql_supplier')->find($sp->id);
|
||||
expect($fresh)->not->toBeNull();
|
||||
expect($fresh->inactive_since)->not->toBeNull();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user