ci(stage5-investigate): round 3 schema discovery — list columns of activity_log/balance_transactions/supplier_projects/supplier_leads; SELECT * on broken audit rows (ids 597-601 + 460-464) and stuck supplier_leads (1110, 1157) + sample failed_webhook_jobs raw_payload + all B1 supplier_projects
This commit is contained in:
Дмитрий
@@ -1,10 +1,8 @@
|
||||
name: Stage 5 day 1 investigation — round 2 (root-cause deep-dive)
|
||||
name: Stage 5 day 1 investigation — round 3 (schema + full rows)
|
||||
|
||||
# Round 2: после round 1 нашли — audit-chain сломан на партициях май
|
||||
# (activity_log + balance_transactions), webhook storm 256k записей все
|
||||
# с reason "B1 does not support SMS". Здесь — root-cause queries.
|
||||
#
|
||||
# READ-ONLY. Запускать через workflow_dispatch вручную.
|
||||
# Round 3: реальные имена колонок hash в audit-таблицах,
|
||||
# реальные имена FK в supplier_projects/supplier_leads,
|
||||
# полное содержимое битых строк (599/462) и застрявших лидов (1110/1157).
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -26,79 +24,63 @@ jobs:
|
||||
chmod 600 ~/.ssh/liderra_deploy
|
||||
ssh-keyscan -H ${{ env.LIDERRA_HOST }} >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
- name: Round 2 root-cause queries
|
||||
- name: Round 3 schema + rows
|
||||
run: |
|
||||
ssh -i ~/.ssh/liderra_deploy ${{ env.LIDERRA_USER }}@${{ env.LIDERRA_HOST }} 'bash -s' <<'REMOTE' | tee /tmp/investigate2.log
|
||||
ssh -i ~/.ssh/liderra_deploy ${{ env.LIDERRA_USER }}@${{ env.LIDERRA_HOST }} 'bash -s' <<'REMOTE' | tee /tmp/investigate3.log
|
||||
set +e
|
||||
cd /var/www/liderra/app
|
||||
|
||||
echo "=========================================="
|
||||
echo "ROOT CAUSE 1 — audit-chain break"
|
||||
echo "SCHEMAS"
|
||||
echo "=========================================="
|
||||
|
||||
echo
|
||||
echo "--- 1.1 Triggers on broken partition activity_log_y2026_m05 ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT trigger_name, event_manipulation, action_timing, action_statement FROM information_schema.triggers WHERE event_object_table='activity_log_y2026_m05';"
|
||||
echo "--- activity_log columns ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT column_name, data_type FROM information_schema.columns WHERE table_name='activity_log' ORDER BY ordinal_position;"
|
||||
|
||||
echo
|
||||
echo "--- 1.2 Triggers on healthy partition activity_log_y2026_m04 (comparison) ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT trigger_name, event_manipulation, action_timing, action_statement FROM information_schema.triggers WHERE event_object_table='activity_log_y2026_m04';"
|
||||
echo "--- balance_transactions columns ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT column_name, data_type FROM information_schema.columns WHERE table_name='balance_transactions' ORDER BY ordinal_position;"
|
||||
|
||||
echo
|
||||
echo "--- 1.3 Triggers on parent activity_log ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT trigger_name, event_manipulation, action_timing FROM information_schema.triggers WHERE event_object_table='activity_log';"
|
||||
echo "--- supplier_projects columns ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT column_name, data_type FROM information_schema.columns WHERE table_name='supplier_projects' ORDER BY ordinal_position;"
|
||||
|
||||
echo
|
||||
echo "--- 1.4 Triggers on broken balance_transactions_y2026_m05 ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT trigger_name, event_manipulation, action_timing FROM information_schema.triggers WHERE event_object_table='balance_transactions_y2026_m05';"
|
||||
|
||||
echo
|
||||
echo "--- 1.5 First broken activity_log row (id=599) context — id 597, 598, 599, 600 ---"
|
||||
sudo -u postgres psql -d liderra -x -c "SELECT id, user_id, action, created_at, prev_hash, current_hash FROM activity_log_y2026_m05 WHERE id IN (597, 598, 599, 600) ORDER BY id;"
|
||||
|
||||
echo
|
||||
echo "--- 1.6 First broken balance_transactions row (id=462) context ---"
|
||||
sudo -u postgres psql -d liderra -x -c "SELECT id, tenant_id, type, created_at, prev_hash, current_hash FROM balance_transactions_y2026_m05 WHERE id IN (460, 461, 462, 463) ORDER BY id;"
|
||||
|
||||
echo
|
||||
echo "--- 1.7 All trigger functions named *chain_hash* ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT proname, pronamespace::regnamespace AS schema FROM pg_proc WHERE proname ILIKE '%chain%' OR proname ILIKE '%audit_hash%';"
|
||||
echo "--- supplier_leads columns ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT column_name, data_type FROM information_schema.columns WHERE table_name='supplier_leads' ORDER BY ordinal_position;"
|
||||
|
||||
|
||||
echo
|
||||
echo
|
||||
echo "=========================================="
|
||||
echo "ROOT CAUSE 2 — webhook storm B1+SMS"
|
||||
echo "BROKEN ROWS — full SELECT *"
|
||||
echo "=========================================="
|
||||
|
||||
echo
|
||||
echo "--- 2.1 Top 10 supplier_lead_id from raw_payload of failed_webhook_jobs last 24h ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT raw_payload->>'supplier_lead_id' AS sl_id, COUNT(*) AS cnt FROM failed_webhook_jobs WHERE failed_at > NOW() - INTERVAL '24 hours' GROUP BY 1 ORDER BY 2 DESC LIMIT 10;"
|
||||
echo "--- activity_log_y2026_m05 ids 597-601 ---"
|
||||
sudo -u postgres psql -d liderra -x -c "SELECT * FROM activity_log_y2026_m05 WHERE id BETWEEN 597 AND 601 ORDER BY id;"
|
||||
|
||||
echo
|
||||
echo "--- 2.2 supplier_leads context for first failing sl_id ---"
|
||||
sudo -u postgres psql -d liderra -x -c "SELECT id, signal_type, signal_identifier, sms_sender, sms_keyword, raw_payload, created_at FROM supplier_leads WHERE id = (SELECT (raw_payload->>'supplier_lead_id')::bigint FROM failed_webhook_jobs WHERE failed_at > NOW() - INTERVAL '24 hours' AND exception ILIKE '%B1%' ORDER BY failed_at DESC LIMIT 1);"
|
||||
echo "--- balance_transactions_y2026_m05 ids 460-464 ---"
|
||||
sudo -u postgres psql -d liderra -x -c "SELECT * FROM balance_transactions_y2026_m05 WHERE id BETWEEN 460 AND 464 ORDER BY id;"
|
||||
|
||||
|
||||
echo
|
||||
echo "--- 2.3 supplier_projects with B1 platform AND SMS signal (illegal combo) ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT id, project_id, platform, signal_type, signal_identifier, sms_sender, sms_keyword, status, current_limit FROM supplier_projects WHERE platform='B1' AND signal_type='sms';" 2>&1 | head -20
|
||||
echo "=========================================="
|
||||
echo "STUCK LEADS 1110 + 1157"
|
||||
echo "=========================================="
|
||||
|
||||
echo
|
||||
echo "--- 2.4 CHECK constraint definition ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT conname, pg_get_constraintdef(oid) FROM pg_constraint WHERE conname='chk_supplier_projects_b1_not_for_sms';"
|
||||
echo "--- supplier_leads.id IN (1110, 1157) ---"
|
||||
sudo -u postgres psql -d liderra -x -c "SELECT * FROM supplier_leads WHERE id IN (1110, 1157);"
|
||||
|
||||
echo
|
||||
echo "--- 2.5 Top project_id concentrating webhook failures ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT sl.matched_project_id, COUNT(DISTINCT fwj.id) AS fails FROM failed_webhook_jobs fwj LEFT JOIN supplier_leads sl ON sl.id = (fwj.raw_payload->>'supplier_lead_id')::bigint WHERE fwj.failed_at > NOW() - INTERVAL '24 hours' GROUP BY 1 ORDER BY 2 DESC LIMIT 10;" 2>&1 | head -20
|
||||
echo "--- failed_webhook_jobs sample raw_payload for sl_id=1110 (1 row) ---"
|
||||
sudo -u postgres psql -d liderra -x -c "SELECT * FROM failed_webhook_jobs WHERE raw_payload->>'supplier_lead_id' = '1110' ORDER BY failed_at DESC LIMIT 1;"
|
||||
|
||||
echo
|
||||
echo "--- 2.6 Raw signal_type distribution of supplier_leads last 24h ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT signal_type, COUNT(*) FROM supplier_leads WHERE created_at > NOW() - INTERVAL '24 hours' GROUP BY 1 ORDER BY 2 DESC;"
|
||||
|
||||
echo
|
||||
echo "--- 2.7 Are real (non-failed) supplier_leads being processed last 24h? ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT COUNT(*) AS total_supplier_leads_24h, COUNT(*) FILTER (WHERE processed_at IS NOT NULL) AS processed, COUNT(*) FILTER (WHERE matched_project_id IS NOT NULL) AS matched FROM supplier_leads WHERE created_at > NOW() - INTERVAL '24 hours';"
|
||||
|
||||
echo "--- All supplier_projects with platform B1 ---"
|
||||
sudo -u postgres psql -d liderra -c "SELECT * FROM supplier_projects WHERE platform='B1' LIMIT 5;"
|
||||
|
||||
echo
|
||||
echo "=========================================="
|
||||
@@ -110,10 +92,10 @@ jobs:
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## Stage 5 day 1 investigation — round 2"
|
||||
echo "## Stage 5 day 1 investigation — round 3 schemas"
|
||||
echo
|
||||
echo '```'
|
||||
cat /tmp/investigate2.log 2>/dev/null || echo "(no output)"
|
||||
cat /tmp/investigate3.log 2>/dev/null || echo "(no output)"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -121,8 +103,8 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: investigate-day1-round2
|
||||
path: /tmp/investigate2.log
|
||||
name: investigate-day1-round3
|
||||
path: /tmp/investigate3.log
|
||||
|
||||
- name: Cleanup SSH key
|
||||
if: always()
|
||||
|
||||
Reference in New Issue
Block a user