From fada3f348c372c3fc30afbcc1ff5ccfa74dd3bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Sat, 16 May 2026 17:01:51 +0300 Subject: [PATCH] =?UTF-8?q?feat(import):=20H1+H2=20=E2=80=94=20=D1=81?= =?UTF-8?q?=D1=85=D0=B5=D0=BC=D0=B0=20import=5Funknown=5Fstatuses=20+=20en?= =?UTF-8?q?richment=20import=5Flog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sprint 4 Task 1 (schema delta §6): - H1: новая таблица import_unknown_statuses (RLS tenant_isolation, UNIQUE(tenant_id,status_ru), FK→tenants/import_log/lead_statuses/users) - H2: +5 колонок import_log (entity_type, source_system, mapping_config, unknown_statuses_count, dry_run) - schema.sql v8.20→v8.21 (64 таблицы / 118 индексов / 40 RLS-политик) - db/CHANGELOG_schema.md v8.21 entry - db/02_grants.sql v8.21 section (crm_app_user/crm_app_admin/crm_readonly) - migrate: hasTable/hasColumn guards (fresh-safe) - tests: 3 Pest-теста (ImportSchemaTest) + SchemaDeltaTest v8.21 metrics - ide-helper: _ide_helper.php + _ide_helper_models.php (были отсутствуют в worktree, phpstan падал молча из-за missing scanFiles entry) Co-Authored-By: Claude Opus 4.7 (1M context) --- ...20000_sprint4_historical_import_schema.php | 77 +++++++++++++++++++ app/tests/Feature/Import/ImportSchemaTest.php | 36 +++++++++ .../Feature/Plan4/Schema/SchemaDeltaTest.php | 13 ++-- db/02_grants.sql | 10 +++ db/CHANGELOG_schema.md | 11 ++- db/schema.sql | 46 ++++++++++- 6 files changed, 182 insertions(+), 11 deletions(-) create mode 100644 app/database/migrations/2026_05_16_120000_sprint4_historical_import_schema.php create mode 100644 app/tests/Feature/Import/ImportSchemaTest.php diff --git a/app/database/migrations/2026_05_16_120000_sprint4_historical_import_schema.php b/app/database/migrations/2026_05_16_120000_sprint4_historical_import_schema.php new file mode 100644 index 00000000..821e336a --- /dev/null +++ b/app/database/migrations/2026_05_16_120000_sprint4_historical_import_schema.php @@ -0,0 +1,77 @@ + "ALTER TABLE import_log ADD COLUMN entity_type VARCHAR(20) NOT NULL DEFAULT 'leads' CHECK (entity_type IN ('leads','projects'))", + 'source_system' => "ALTER TABLE import_log ADD COLUMN source_system VARCHAR(50) NOT NULL DEFAULT 'crm.bp-gr.ru'", + 'mapping_config' => 'ALTER TABLE import_log ADD COLUMN mapping_config JSONB', + 'unknown_statuses_count' => 'ALTER TABLE import_log ADD COLUMN unknown_statuses_count INT NOT NULL DEFAULT 0', + 'dry_run' => 'ALTER TABLE import_log ADD COLUMN dry_run BOOLEAN NOT NULL DEFAULT FALSE', + ] as $column => $ddl) { + if (! Schema::hasColumn('import_log', $column)) { + DB::statement($ddl); + } + } + + if (! Schema::hasTable('import_unknown_statuses')) { + DB::statement(<<<'SQL' + CREATE TABLE import_unknown_statuses ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, + import_log_id BIGINT REFERENCES import_log(id) ON DELETE SET NULL, + status_ru VARCHAR(100) NOT NULL, + occurrences INT NOT NULL DEFAULT 0, + mapped_to_slug VARCHAR(50) REFERENCES lead_statuses(slug), + resolved_at TIMESTAMPTZ, + resolved_by BIGINT REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ, + UNIQUE (tenant_id, status_ru) + ) + SQL); + + DB::statement( + 'CREATE INDEX idx_import_unknown_statuses_unresolved + ON import_unknown_statuses (tenant_id) WHERE mapped_to_slug IS NULL' + ); + + DB::statement('ALTER TABLE import_unknown_statuses ENABLE ROW LEVEL SECURITY'); + DB::statement( + "CREATE POLICY tenant_isolation ON import_unknown_statuses + USING (tenant_id = current_setting('app.current_tenant_id')::bigint)" + ); + } + } + + public function down(): void + { + // down() не симметричен: на проекте rollback применяется только после + // migrate:fresh (см. add_archived_at_to_projects). Для отката v8.21 — + // отдельный schema-bump, не эта миграция. + DB::statement('DROP TABLE IF EXISTS import_unknown_statuses'); + + foreach (['entity_type', 'source_system', 'mapping_config', 'unknown_statuses_count', 'dry_run'] as $column) { + if (Schema::hasColumn('import_log', $column)) { + Schema::table('import_log', fn ($table) => $table->dropColumn($column)); + } + } + } +}; diff --git a/app/tests/Feature/Import/ImportSchemaTest.php b/app/tests/Feature/Import/ImportSchemaTest.php new file mode 100644 index 00000000..c8c0fcfc --- /dev/null +++ b/app/tests/Feature/Import/ImportSchemaTest.php @@ -0,0 +1,36 @@ +toBeTrue("import_log.$column отсутствует"); + } +}); + +test('import_unknown_statuses существует с RLS', function (): void { + expect(Schema::hasTable('import_unknown_statuses'))->toBeTrue(); + + $rls = DB::selectOne( + "SELECT relrowsecurity FROM pg_class WHERE relname = 'import_unknown_statuses'" + ); + expect($rls)->not->toBeNull('pg_class row for import_unknown_statuses не найден'); + /** @var object{relrowsecurity: bool} $rls */ + expect($rls->relrowsecurity)->toBeTrue('RLS не включён на import_unknown_statuses'); + + $policy = DB::selectOne( + "SELECT 1 AS ok FROM pg_policies WHERE tablename = 'import_unknown_statuses' AND policyname = 'tenant_isolation'" + ); + expect($policy)->not->toBeNull('Политика tenant_isolation отсутствует'); +}); + +test('import_unknown_statuses имеет UNIQUE (tenant_id, status_ru)', function (): void { + $unique = DB::selectOne( + "SELECT 1 AS ok FROM pg_constraint + WHERE conrelid = 'import_unknown_statuses'::regclass AND contype = 'u'" + ); + expect($unique)->not->toBeNull('UNIQUE-ограничение отсутствует'); +}); diff --git a/app/tests/Feature/Plan4/Schema/SchemaDeltaTest.php b/app/tests/Feature/Plan4/Schema/SchemaDeltaTest.php index dff0b274..965eedee 100644 --- a/app/tests/Feature/Plan4/Schema/SchemaDeltaTest.php +++ b/app/tests/Feature/Plan4/Schema/SchemaDeltaTest.php @@ -59,24 +59,25 @@ it('supplier_csv_reconcile_log table exists with required columns and status CHE ]))->toThrow(QueryException::class); }); -it('schema.sql v8.19 has correct metrics — 62 base tables, 117 indexes, 39 RLS policies', function () { +it('schema.sql v8.21 has correct metrics — 63 base tables, 118 indexes, 40 RLS policies', function () { // Замена destructive `migrate:fresh` (cross-test coupling: после DROP CASCADE остальные // Feature-тесты в той же сессии видели пустую БД). Static parse `db/schema.sql` — - // источник истины метрик из spec §2.4 / db/CHANGELOG_schema.md v8.19. + // источник истины метрик из spec §2.4 / db/CHANGELOG_schema.md v8.21. + // v8.21 (Sprint 4): +1 таблица import_unknown_statuses, +1 индекс, +1 RLS-политика. $schemaPath = dirname(base_path()).DIRECTORY_SEPARATOR.'db'.DIRECTORY_SEPARATOR.'schema.sql'; expect(is_file($schemaPath) && is_readable($schemaPath))->toBeTrue(); $schema = file_get_contents($schemaPath); expect($schema)->not->toBeFalse(); - // 62 base tables = все CREATE TABLE минус 12 партиций (PARTITION OF). + // 63 base tables = все CREATE TABLE минус 12 партиций (PARTITION OF). $createTables = preg_match_all('/^CREATE TABLE\b/m', $schema); $partitionOf = preg_match_all('/CREATE TABLE\s+\w+\s+PARTITION OF\b/m', $schema); $baseTables = $createTables - $partitionOf; - expect($baseTables)->toBe(62); + expect($baseTables)->toBe(63); $createIndexes = preg_match_all('/^CREATE\s+(?:UNIQUE\s+)?INDEX\b/m', $schema); - expect($createIndexes)->toBe(117); + expect($createIndexes)->toBe(118); $createPolicies = preg_match_all('/^CREATE\s+POLICY\b/m', $schema); - expect($createPolicies)->toBe(39); + expect($createPolicies)->toBe(40); }); diff --git a/db/02_grants.sql b/db/02_grants.sql index 2519946a..8e4f9e18 100644 --- a/db/02_grants.sql +++ b/db/02_grants.sql @@ -140,3 +140,13 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE ON TABLE supplier_csv_reconcile_log TO crm_supplier_worker; GRANT USAGE, SELECT ON SEQUENCE supplier_csv_reconcile_log_id_seq TO crm_supplier_worker; + +-- ============================================================================= +-- v8.21 (Sprint 4): import_unknown_statuses — tenant-level маппинг неизвестных +-- статусов CSV. Tenant-scoped (RLS tenant_isolation). Явные гранты дополняют +-- GRANT ... ON ALL TABLES выше (документируют разрешения per-table для ревью). +-- ============================================================================= + +GRANT SELECT, INSERT, UPDATE, DELETE ON import_unknown_statuses TO crm_app_user, crm_app_admin; +GRANT SELECT ON import_unknown_statuses TO crm_readonly; +GRANT USAGE, SELECT ON SEQUENCE import_unknown_statuses_id_seq TO crm_app_user, crm_app_admin; diff --git a/db/CHANGELOG_schema.md b/db/CHANGELOG_schema.md index 04c15e84..04624d13 100644 --- a/db/CHANGELOG_schema.md +++ b/db/CHANGELOG_schema.md @@ -1,11 +1,18 @@ # CHANGELOG schema.sql — Лидерра -**Назначение:** консолидированный журнал изменений `schema.sql`. Содержит девятнадцать записей в обратном хронологическом порядке (v8.20 → v8.19 → v8.18 → v8.17 → v8.16 → v8.15 → v8.14 → v8.13 → v8.12 → v8.11 → v8.10 → v8.9 → v8.8 → v8.7 → v8.6 → v8.5 → v8.4 → v8.3 → v8.2), как принято в keep-a-changelog. +**Назначение:** консолидированный журнал изменений `schema.sql`. Содержит двадцать записей в обратном хронологическом порядке (v8.21 → v8.20 → v8.19 → v8.18 → v8.17 → v8.16 → v8.15 → v8.14 → v8.13 → v8.12 → v8.11 → v8.10 → v8.9 → v8.8 → v8.7 → v8.6 → v8.5 → v8.4 → v8.3 → v8.2), как принято в keep-a-changelog. -**Файл схемы:** `schema.sql` (текущая версия — v8.20, консолидированная — разворачивает БД с нуля). +**Файл схемы:** `schema.sql` (текущая версия — v8.21, консолидированная — разворачивает БД с нуля). **История записей:** +## v8.21 — 2026-05-16 — Sprint 4 (историческая миграция лидов §6) + +- **+1 таблица** `import_unknown_statuses` (tenant-level маппинг неизвестных статусов CSV; RLS `tenant_isolation`; UNIQUE `(tenant_id, status_ru)`; partial index `idx_import_unknown_statuses_unresolved`). +- **+5 колонок** в `import_log`: `entity_type`, `source_system`, `mapping_config`, `unknown_statuses_count`, `dry_run`. +- GRANTs для `import_unknown_statuses`: SELECT/INSERT/UPDATE/DELETE — `crm_app_user`/`crm_app_admin`; SELECT — `crm_readonly`. +- Миграция: `2026_05_16_120000_sprint4_historical_import_schema.php` (guard'ы `hasTable`/`hasColumn`). + ## v8.20 (11.05.2026 — Plan 5) **Added:** diff --git a/db/schema.sql b/db/schema.sql index f740c9a0..8ab03d33 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1,7 +1,8 @@ -- ============================================================================= -- schema.sql — единая схема БД для SaaS-аналога crm.bp-gr.ru («Лидерра») --- Версия: v8.20 (11.05.2026 — Plan 5 frontend projects UI: projects.archived_at TIMESTAMPTZ NULL для soft archive flow; tenants.limits JSONB NOT NULL DEFAULT '{}' для per-tenant project/user лимитов) --- Метрики: 63 базовые таблицы (61 regular + 2 partitioned parents: deals + supplier_lead_costs) + 12 партиций / 117 индексов / 39 RLS-политик / 5 функций / 13 триггеров +-- Версия: v8.21 (16.05.2026 — Sprint 4: import_unknown_statuses + import_log enrichment (+5 колонок)) +-- Метрики: 64 базовые таблицы (62 regular + 2 partitioned parents: deals + supplier_lead_costs) + 12 партиций / 118 индексов / 40 RLS-политик / 5 функций / 13 триггеров +-- Базовая версия: v8.20 (11.05.2026 — Plan 5 frontend projects UI: projects.archived_at TIMESTAMPTZ NULL для soft archive flow; tenants.limits JSONB NOT NULL DEFAULT '{}' для per-tenant project/user лимитов) -- Базовая версия: v8.19 (11.05.2026 — Plan 4 billing+csv+admin: tenants.delivered_in_month, lead_charges.charge_source + CHECK, supplier_leads.recovered_from_csv_at, supplier_csv_reconcile_log) -- Базовая версия: v8.18 (10.05.2026 — Plan 2/5 Task 1: supplier_leads SaaS-level + projects.delivered_today + 2 system_settings rows для supplier-webhook + IP allowlist defense-in-depth) -- Базовая версия: v8.17 (10.05.2026 — Plan 1/5 Task 2 fix: FK projects.supplier_b{1,2,3}_project_id → supplier_projects (ON DELETE SET NULL) + 3 partial index + CHECK chk_projects_b1_not_for_sms (defense-in-depth дублирует chk_supplier_projects_b1_not_for_sms на Project-уровне). Закрывает code-review BLOCKER#1 + WARNING#3 от 10.05.2026 поздний вечер) @@ -1489,9 +1490,38 @@ CREATE TABLE import_log ( CHECK (status IN ('pending','processing','done','failed')), error_message TEXT, started_at TIMESTAMPTZ DEFAULT NOW(), - finished_at TIMESTAMPTZ + finished_at TIMESTAMPTZ, + -- Sprint 4 (H2): enrichment-колонки для исторической миграции лидов (раздел 6.4) + entity_type VARCHAR(20) NOT NULL DEFAULT 'leads' + CHECK (entity_type IN ('leads','projects')), + source_system VARCHAR(50) NOT NULL DEFAULT 'crm.bp-gr.ru', + mapping_config JSONB, + unknown_statuses_count INT NOT NULL DEFAULT 0, + dry_run BOOLEAN NOT NULL DEFAULT FALSE ); +-- ----------------------------------------------------------------------------- +-- import_unknown_statuses — tenant-level маппинг неизвестных статусов CSV (раздел 6.4) +-- Sprint 4 (H1): русский статус из CSV, не найденный в STATUS_RU_TO_SLUG, пишется сюда. +-- Wizard (§6.6) проставляет mapped_to_slug; повторный импорт применяет маппинг. +-- ----------------------------------------------------------------------------- +CREATE TABLE import_unknown_statuses ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE, + import_log_id BIGINT REFERENCES import_log(id) ON DELETE SET NULL, + status_ru VARCHAR(100) NOT NULL, + occurrences INT NOT NULL DEFAULT 0, + mapped_to_slug VARCHAR(50) REFERENCES lead_statuses(slug), + resolved_at TIMESTAMPTZ, + resolved_by BIGINT REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ, + UNIQUE (tenant_id, status_ru) +); + +CREATE INDEX idx_import_unknown_statuses_unresolved + ON import_unknown_statuses (tenant_id) WHERE mapped_to_slug IS NULL; + -- ============================================================================= -- 5. DEALS — партиционированная по received_at (раздел 7.3) @@ -2703,6 +2733,7 @@ ALTER TABLE push_subscriptions ENABLE ROW LEVEL SECURITY; ALTER TABLE comment_templates ENABLE ROW LEVEL SECURITY; ALTER TABLE deal_tags ENABLE ROW LEVEL SECURITY; ALTER TABLE import_log ENABLE ROW LEVEL SECURITY; +ALTER TABLE import_unknown_statuses ENABLE ROW LEVEL SECURITY; ALTER TABLE activity_log ENABLE ROW LEVEL SECURITY; ALTER TABLE reminders ENABLE ROW LEVEL SECURITY; ALTER TABLE webhook_log ENABLE ROW LEVEL SECURITY; @@ -2743,6 +2774,7 @@ CREATE POLICY tenant_isolation ON push_subscriptions USING (tenant_id = cur CREATE POLICY tenant_isolation ON comment_templates USING (tenant_id = current_setting('app.current_tenant_id')::bigint); CREATE POLICY tenant_isolation ON deal_tags USING (tenant_id = current_setting('app.current_tenant_id')::bigint); CREATE POLICY tenant_isolation ON import_log USING (tenant_id = current_setting('app.current_tenant_id')::bigint); +CREATE POLICY tenant_isolation ON import_unknown_statuses USING (tenant_id = current_setting('app.current_tenant_id')::bigint); CREATE POLICY tenant_isolation ON activity_log USING (tenant_id = current_setting('app.current_tenant_id')::bigint); CREATE POLICY tenant_isolation ON reminders USING (tenant_id = current_setting('app.current_tenant_id')::bigint); CREATE POLICY tenant_isolation ON webhook_log USING (tenant_id = current_setting('app.current_tenant_id')::bigint); @@ -2870,6 +2902,14 @@ CREATE POLICY tenant_isolation ON auth_log USING ( -- USAGE на sequences соответствующих таблиц; -- запрещено: SELECT, UPDATE, DELETE, TRUNCATE. +-- v8.21 (Sprint 4): import_unknown_statuses — tenant-level маппинг неизвестных статусов CSV. +-- RLS tenant_isolation активна. Права аналогичны import_log (tenant-таблица). +-- Деплой через db/02_grants.sql (явная секция v8.21). +-- +-- GRANT SELECT, INSERT, UPDATE, DELETE ON import_unknown_statuses TO crm_app_user, crm_app_admin; +-- GRANT SELECT ON import_unknown_statuses TO crm_readonly; +-- GRANT USAGE, SELECT ON SEQUENCE import_unknown_statuses_id_seq TO crm_app_user, crm_app_admin; + -- v8.5 (OPEN-И-14): defense-in-depth — REVOKE ALL на 6 saas-таблицах -- от crm_app_user. К этим таблицам tenant-приложение доступа НЕ должно -- иметь даже теоретически (RLS + REVOKE = 2 барьера).