ci: stage5 day-1 investigation workflow — diagnose audit:verify-chains failures + failed_webhook_jobs 163k spike (one-shot read-only, hardcoded SQL on incidents_log/failed_jobs/failed_webhook_jobs + direct audit:verify-chains -v artisan call)

2026-05-29 06:24:30 +03:00
parent 5c02d33cce
commit ea7cc84a37
2 changed files with 121 additions and 11 deletions
@@ -0,0 +1,110 @@
+name: Stage 5 day 1 investigation — audit-chains + webhook-failures
+
+# Одноразовая диагностика двух сигналов, пойманных в день 1 мониторинга
+# этапа 5 (29.05.2026 ~05:43 МСК):
+#   1. scheduler_heartbeats: audit:verify-chains consecutive_failures=4
+#   2. failed_webhook_jobs за 24ч: 163417 записей (порог тревоги 100)
+#
+# Hardcoded queries — нет user-input'а, whitelist не нужен.
+# READ-ONLY. Запускать через workflow_dispatch вручную.
+
+on:
+  workflow_dispatch:
+
+jobs:
+  investigate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    env:
+      LIDERRA_HOST: 111.88.246.137
+      LIDERRA_USER: ubuntu
+
+    steps:
+      - name: Setup SSH key
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.LIDERRA_SSH_KEY }}" > ~/.ssh/liderra_deploy
+          chmod 600 ~/.ssh/liderra_deploy
+          ssh-keyscan -H ${{ env.LIDERRA_HOST }} >> ~/.ssh/known_hosts 2>/dev/null
+
+      - name: Run diagnostic queries on prod
+        run: |
+          ssh -i ~/.ssh/liderra_deploy ${{ env.LIDERRA_USER }}@${{ env.LIDERRA_HOST }} 'bash -s' <<'REMOTE' | tee /tmp/investigate.log
+            set +e
+            cd /var/www/liderra/app
+
+            echo "=========================================="
+            echo "SIGNAL 1 — audit:verify-chains failing"
+            echo "=========================================="
+
+            echo
+            echo "--- 1.1 scheduler_heartbeats full row ---"
+            sudo -u postgres psql -d liderra -x -c "SELECT * FROM scheduler_heartbeats WHERE command_name='audit:verify-chains';"
+
+            echo
+            echo "--- 1.2 incidents_log entries mentioning audit/chain ---"
+            sudo -u postgres psql -d liderra -c "SELECT id, severity, created_at, LEFT(root_cause, 120) AS root_cause FROM incidents_log WHERE root_cause ILIKE '%audit%' OR root_cause ILIKE '%chain%' OR root_cause ILIKE '%verify%' ORDER BY created_at DESC LIMIT 10;"
+
+            echo
+            echo "--- 1.3 failed_jobs entries for VerifyAuditChains command ---"
+            sudo -u postgres psql -d liderra -c "SELECT id, failed_at, LEFT(exception, 200) AS exception FROM failed_jobs WHERE payload::text ILIKE '%audit%' OR payload::text ILIKE '%verify-chains%' OR exception ILIKE '%audit%' OR exception ILIKE '%chain%' ORDER BY failed_at DESC LIMIT 5;"
+
+            echo
+            echo "--- 1.4 Direct artisan call: audit:verify-chains ---"
+            sudo -u www-data php artisan audit:verify-chains -v 2>&1 | head -50
+            echo "Exit: $?"
+
+            echo
+            echo
+            echo "=========================================="
+            echo "SIGNAL 2 — failed_webhook_jobs 163k in 24h"
+            echo "=========================================="
+
+            echo
+            echo "--- 2.1 Total count ever ---"
+            sudo -u postgres psql -d liderra -tA -c "SELECT COUNT(*) FROM failed_webhook_jobs;"
+
+            echo
+            echo "--- 2.2 Time distribution by hour, last 48h ---"
+            sudo -u postgres psql -d liderra -c "SELECT DATE_TRUNC('hour', failed_at AT TIME ZONE 'UTC') AS hr_utc, COUNT(*) AS cnt FROM failed_webhook_jobs WHERE failed_at > NOW() - INTERVAL '48 hours' GROUP BY 1 ORDER BY 1 DESC LIMIT 48;"
+
+            echo
+            echo "--- 2.3 Exception distribution top 10 ---"
+            sudo -u postgres psql -d liderra -c "SELECT LEFT(exception, 100) AS reason, COUNT(*) AS cnt FROM failed_webhook_jobs WHERE failed_at > NOW() - INTERVAL '24 hours' GROUP BY 1 ORDER BY 2 DESC LIMIT 10;"
+
+            echo
+            echo "--- 2.4 Schema of failed_webhook_jobs (column names) ---"
+            sudo -u postgres psql -d liderra -c "SELECT column_name FROM information_schema.columns WHERE table_name='failed_webhook_jobs' ORDER BY ordinal_position;"
+
+            echo
+            echo "--- 2.5 Sample 3 most recent rows ---"
+            sudo -u postgres psql -d liderra -x -c "SELECT * FROM failed_webhook_jobs ORDER BY failed_at DESC LIMIT 3;"
+
+            echo
+            echo "=========================================="
+            echo "DONE"
+            echo "=========================================="
+          REMOTE
+
+      - name: Print summary
+        if: always()
+        run: |
+          {
+            echo "## Stage 5 day 1 investigation"
+            echo
+            echo '```'
+            cat /tmp/investigate.log 2>/dev/null || echo "(no output)"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: investigate-day1
+          path: /tmp/investigate.log
+
+      - name: Cleanup SSH key
+        if: always()
+        run: rm -f ~/.ssh/liderra_deploy