From ea7cc84a372682e01bd21dc2a869dc55b47534d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Fri, 29 May 2026 06:24:30 +0300 Subject: [PATCH] =?UTF-8?q?ci:=20stage5=20day-1=20investigation=20workflow?= =?UTF-8?q?=20=E2=80=94=20diagnose=20audit:verify-chains=20failures=20+=20?= =?UTF-8?q?failed=5Fwebhook=5Fjobs=20163k=20spike=20(one-shot=20read-only,?= =?UTF-8?q?=20hardcoded=20SQL=20on=20incidents=5Flog/failed=5Fjobs/failed?= =?UTF-8?q?=5Fwebhook=5Fjobs=20+=20direct=20audit:verify-chains=20-v=20art?= =?UTF-8?q?isan=20call)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/stage5-investigate-day1.yml | 110 ++++++++++++++++++ docs/observer/STATUS.md | 22 ++-- 2 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/stage5-investigate-day1.yml diff --git a/.github/workflows/stage5-investigate-day1.yml b/.github/workflows/stage5-investigate-day1.yml new file mode 100644 index 00000000..c654a777 --- /dev/null +++ b/.github/workflows/stage5-investigate-day1.yml @@ -0,0 +1,110 @@ +name: Stage 5 day 1 investigation — audit-chains + webhook-failures + +# Одноразовая диагностика двух сигналов, пойманных в день 1 мониторинга +# этапа 5 (29.05.2026 ~05:43 МСК): +# 1. scheduler_heartbeats: audit:verify-chains consecutive_failures=4 +# 2. failed_webhook_jobs за 24ч: 163417 записей (порог тревоги 100) +# +# Hardcoded queries — нет user-input'а, whitelist не нужен. +# READ-ONLY. Запускать через workflow_dispatch вручную. + +on: + workflow_dispatch: + +jobs: + investigate: + runs-on: ubuntu-latest + timeout-minutes: 10 + + env: + LIDERRA_HOST: 111.88.246.137 + LIDERRA_USER: ubuntu + + steps: + - name: Setup SSH key + run: | + mkdir -p ~/.ssh + echo "${{ secrets.LIDERRA_SSH_KEY }}" > ~/.ssh/liderra_deploy + chmod 600 ~/.ssh/liderra_deploy + ssh-keyscan -H ${{ env.LIDERRA_HOST }} >> ~/.ssh/known_hosts 2>/dev/null + + - name: Run diagnostic queries on prod + run: | + ssh -i ~/.ssh/liderra_deploy ${{ env.LIDERRA_USER }}@${{ env.LIDERRA_HOST }} 'bash -s' <<'REMOTE' | tee /tmp/investigate.log + set +e + cd /var/www/liderra/app + + echo "==========================================" + echo "SIGNAL 1 — audit:verify-chains failing" + echo "==========================================" + + echo + echo "--- 1.1 scheduler_heartbeats full row ---" + sudo -u postgres psql -d liderra -x -c "SELECT * FROM scheduler_heartbeats WHERE command_name='audit:verify-chains';" + + echo + echo "--- 1.2 incidents_log entries mentioning audit/chain ---" + sudo -u postgres psql -d liderra -c "SELECT id, severity, created_at, LEFT(root_cause, 120) AS root_cause FROM incidents_log WHERE root_cause ILIKE '%audit%' OR root_cause ILIKE '%chain%' OR root_cause ILIKE '%verify%' ORDER BY created_at DESC LIMIT 10;" + + echo + echo "--- 1.3 failed_jobs entries for VerifyAuditChains command ---" + sudo -u postgres psql -d liderra -c "SELECT id, failed_at, LEFT(exception, 200) AS exception FROM failed_jobs WHERE payload::text ILIKE '%audit%' OR payload::text ILIKE '%verify-chains%' OR exception ILIKE '%audit%' OR exception ILIKE '%chain%' ORDER BY failed_at DESC LIMIT 5;" + + echo + echo "--- 1.4 Direct artisan call: audit:verify-chains ---" + sudo -u www-data php artisan audit:verify-chains -v 2>&1 | head -50 + echo "Exit: $?" + + echo + echo + echo "==========================================" + echo "SIGNAL 2 — failed_webhook_jobs 163k in 24h" + echo "==========================================" + + echo + echo "--- 2.1 Total count ever ---" + sudo -u postgres psql -d liderra -tA -c "SELECT COUNT(*) FROM failed_webhook_jobs;" + + echo + echo "--- 2.2 Time distribution by hour, last 48h ---" + sudo -u postgres psql -d liderra -c "SELECT DATE_TRUNC('hour', failed_at AT TIME ZONE 'UTC') AS hr_utc, COUNT(*) AS cnt FROM failed_webhook_jobs WHERE failed_at > NOW() - INTERVAL '48 hours' GROUP BY 1 ORDER BY 1 DESC LIMIT 48;" + + echo + echo "--- 2.3 Exception distribution top 10 ---" + sudo -u postgres psql -d liderra -c "SELECT LEFT(exception, 100) AS reason, COUNT(*) AS cnt FROM failed_webhook_jobs WHERE failed_at > NOW() - INTERVAL '24 hours' GROUP BY 1 ORDER BY 2 DESC LIMIT 10;" + + echo + echo "--- 2.4 Schema of failed_webhook_jobs (column names) ---" + sudo -u postgres psql -d liderra -c "SELECT column_name FROM information_schema.columns WHERE table_name='failed_webhook_jobs' ORDER BY ordinal_position;" + + echo + echo "--- 2.5 Sample 3 most recent rows ---" + sudo -u postgres psql -d liderra -x -c "SELECT * FROM failed_webhook_jobs ORDER BY failed_at DESC LIMIT 3;" + + echo + echo "==========================================" + echo "DONE" + echo "==========================================" + REMOTE + + - name: Print summary + if: always() + run: | + { + echo "## Stage 5 day 1 investigation" + echo + echo '```' + cat /tmp/investigate.log 2>/dev/null || echo "(no output)" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: investigate-day1 + path: /tmp/investigate.log + + - name: Cleanup SSH key + if: always() + run: rm -f ~/.ssh/liderra_deploy diff --git a/docs/observer/STATUS.md b/docs/observer/STATUS.md index 28dd49a8..4070bd93 100644 --- a/docs/observer/STATUS.md +++ b/docs/observer/STATUS.md @@ -1,6 +1,6 @@ # Brain Status (auto-generated) -Last updated: 2026-05-29T02:24:47.886Z +Last updated: 2026-05-29T02:42:37.928Z | Контролёр | Состояние | Детали | |---|---|---| @@ -8,13 +8,13 @@ Last updated: 2026-05-29T02:24:47.886Z | C2 Cross-ref consistency | ✅ | [cross-ref-checker] OK — 0 drift in 4 files | | C3 Observer-of-observer | ✅ | [observer-of-observer] OK — last read 0 week(s) ago | | C4 Сигнальный статус | ✅ | This file (self-reference) | -| C5 Observer-coverage | ⚠️ | 660 episode(s) this month · Stop-hook + post-commit OK · 20 missed activation(s) — see /brain-retro | +| C5 Observer-coverage | ⚠️ | 664 episode(s) this month · Stop-hook + post-commit OK · 20 missed activation(s) — see /brain-retro | | C6 Chain map sync | ✅ | [chain-map-checker] OK — 16 chains in sync | ## Метрики (информационные, не алерты) -- Observer evidence: 660 episodes this month, 0 observer_error markers, 131 PII matches before filter -- Legacy v1 episodes (not in factor analysis): 521 +- Observer evidence: 664 episodes this month, 0 observer_error markers, 131 PII matches before filter +- Legacy v1 episodes (not in factor analysis): 525 - Last /brain-retro: 2 day(s) ago - Использование узлов: см. `/brain-retro` (раз в спринт). missed_activations: 20. **Неиспользованные узлы — не алерт, если профильной задачи не было** (Pravila §16.4 v1.36; capability-readiness; см. memory `feedback_brain_unused_tools_not_problem` — outside-repo memory store). @@ -31,9 +31,9 @@ Baseline дисциплины роутера (этап 2 router discipline overh | cleanup | 6 | 0.0% | 0.0% | | refactor | 1 | 0.0% | 0.0% | -Router step distribution: 1: 292, 2: 236, 3: 63, 5: 60 +Router step distribution: 1: 296, 2: 236, 3: 63, 5: 60 -Boundaries applied (ADR / границы): 75 of 651 эпизодов (11.5%). +Boundaries applied (ADR / границы): 75 of 655 эпизодов (11.5%). ## Активные многоэтапные проекты @@ -67,7 +67,7 @@ Episodes since last run: 542 / threshold: 10 ## Reviewer: субагент vs fallback -0 эпизодов проверено из 660. +0 эпизодов проверено из 664. ## Reviewer findings @@ -109,9 +109,9 @@ Episodes since last run: 542 / threshold: 10 | Фраза | За всё время | За сегодня | |---|---|---| -| `recovery` | 901 | 4 | -| `ремонт инфраструктуры` | 220 | 35 ⚠️ | -| `без скилов` | 185 | 7 ⚠️ | +| `recovery` | 914 | 17 ⚠️ | +| `ремонт инфраструктуры` | 221 | 36 ⚠️ | +| `без скилов` | 201 | 23 ⚠️ | | `срочно` | 93 | 0 | | `memory dump` | 17 | 0 | | `direct ok` | 6 | 0 | @@ -123,7 +123,7 @@ Episodes since last run: 542 / threshold: 10 | PID | Имя | CPU-время | Возраст | |---|---|---|---| -| 3464 | MsMpEng | 1.04ч | NaNч | +| 3464 | MsMpEng | 1.05ч | NaNч | ⚠️ Проверь, не «осиротевшие» ли это процессы от завершённых Claude-сессий.