feat(stage5): daily monitor workflow + remove non-existent partitions:list from artisan-run whitelist + checklist refinement (GitHub-cron 06:00 UTC daily 29.05-04.06 runs scheduler:check-heartbeats + incidents:watch-failures + migrate:status + 4 SQL signals from incidents_log/project_routing_snapshots/failed_webhook_jobs/scheduler_heartbeats; window auto-stops after 2026-06-05; result to job summary + artifact)
This commit is contained in:
Дмитрий
@@ -45,7 +45,7 @@ jobs:
|
||||
echo "Requested: '$CMD_TRIM'"
|
||||
|
||||
# Group 1 — read-only / dry-run / inspection: всегда разрешены
|
||||
READ_ONLY_RE='^(migrate:status|route:list|schedule:list|queue:listen --help|about|env:show|config:show|cache:table|view:cache|optimize:status|snapshot:backfill( --date=20[2-9][0-9]-[0-1][0-9]-[0-3][0-9])?|scheduler:check-heartbeats|incidents:watch-failures( --threshold-spike=[0-9]+)?( --threshold-daily=[0-9]+)?( --persistent-hours=[0-9]+)?|supplier:rekey-orphans --dry-run|audit:verify-chains|partitions:list)( *)$'
|
||||
READ_ONLY_RE='^(migrate:status|route:list|schedule:list|queue:listen --help|about|env:show|config:show|cache:table|view:cache|optimize:status|snapshot:backfill( --date=20[2-9][0-9]-[0-1][0-9]-[0-3][0-9])?|scheduler:check-heartbeats|incidents:watch-failures( --threshold-spike=[0-9]+)?( --threshold-daily=[0-9]+)?( --persistent-hours=[0-9]+)?|supplier:rekey-orphans --dry-run|audit:verify-chains)( *)$'
|
||||
|
||||
# Group 2 — mutating: требуют confirm_apply=true
|
||||
MUTATING_RE='^(supplier:rekey-orphans|cache:clear|view:clear|config:clear|route:clear|optimize:clear|optimize|queue:restart|partitions:create-months( --months=[0-9]+)?|partitions:drop-old)( *)$'
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
name: Stage 5 daily monitor (29.05→04.06)
|
||||
|
||||
# Автоматический ежедневный мониторинг 3 ключевых сигналов прода
|
||||
# во время 7-дневного окна перед переключением supplier_export_mode
|
||||
# online→batch (Stage 5 Task 5.1).
|
||||
#
|
||||
# Запускается GitHub-cron'ом каждое утро 06:00 UTC (09:00 МСК)
|
||||
# 29.05.2026 — 04.06.2026 (после 04.06 workflow можно отключить
|
||||
# через UI Actions tab → Disable workflow, либо удалить файл).
|
||||
# Также доступен ручной запуск через workflow_dispatch.
|
||||
#
|
||||
# Выводит результаты в job summary + сохраняет как artifact.
|
||||
#
|
||||
# План мониторинга:
|
||||
# docs/superpowers/plans/2026-05-29-stage5-monitoring-checklist.md
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 06:00 UTC = 09:00 МСК ежедневно
|
||||
- cron: '0 6 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
monitor:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
# Жёсткий стоп — workflow ничего не делает после 04.06.2026 даже
|
||||
# если кто-то забудет отключить. CRON в GitHub Actions не имеет
|
||||
# "until date" — реализуем через if-check на runner side.
|
||||
if: github.event_name == 'workflow_dispatch' || github.event.schedule == '0 6 * * *'
|
||||
|
||||
env:
|
||||
LIDERRA_HOST: 111.88.246.137
|
||||
LIDERRA_USER: ubuntu
|
||||
|
||||
steps:
|
||||
- name: Check window not expired
|
||||
id: window
|
||||
run: |
|
||||
TODAY=$(date -u +%Y-%m-%d)
|
||||
DEADLINE='2026-06-05' # 04.06 + 1 день grace
|
||||
if [[ "$TODAY" > "$DEADLINE" ]]; then
|
||||
echo "::notice::Stage 5 monitoring window closed at $DEADLINE. Disable this workflow via Actions UI."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Setup SSH key
|
||||
if: steps.window.outputs.skip != 'true'
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
echo "${{ secrets.LIDERRA_SSH_KEY }}" > ~/.ssh/liderra_deploy
|
||||
chmod 600 ~/.ssh/liderra_deploy
|
||||
ssh-keyscan -H ${{ env.LIDERRA_HOST }} >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
- name: Run 3 checks
|
||||
if: steps.window.outputs.skip != 'true'
|
||||
run: |
|
||||
ssh -i ~/.ssh/liderra_deploy ${{ env.LIDERRA_USER }}@${{ env.LIDERRA_HOST }} 'bash -s' <<'REMOTE' | tee /tmp/monitor.log
|
||||
set +e
|
||||
cd /var/www/liderra/app
|
||||
echo "=== Date: $(date -u) ==="
|
||||
|
||||
echo
|
||||
echo "=== 1. scheduler:check-heartbeats ==="
|
||||
sudo -u www-data php artisan scheduler:check-heartbeats 2>&1
|
||||
echo "Exit: $?"
|
||||
|
||||
echo
|
||||
echo "=== 2. incidents:watch-failures ==="
|
||||
sudo -u www-data php artisan incidents:watch-failures 2>&1
|
||||
echo "Exit: $?"
|
||||
|
||||
echo
|
||||
echo "=== 3. migrate:status ==="
|
||||
sudo -u www-data php artisan migrate:status 2>&1 | tail -8
|
||||
echo "Exit: $?"
|
||||
|
||||
echo
|
||||
echo "=== Auxiliary signals from system tables ==="
|
||||
echo "--- last 3 incidents_log entries ---"
|
||||
sudo -u postgres psql -d liderra -tA -c "SELECT severity, created_at, root_cause FROM incidents_log ORDER BY created_at DESC LIMIT 3;" 2>&1
|
||||
echo "--- snapshot count last 3 days ---"
|
||||
sudo -u postgres psql -d liderra -tA -c "SELECT snapshot_date, COUNT(*) FROM project_routing_snapshots GROUP BY 1 ORDER BY 1 DESC LIMIT 3;" 2>&1
|
||||
echo "--- failed_webhook_jobs last 24h count ---"
|
||||
sudo -u postgres psql -d liderra -tA -c "SELECT COUNT(*) FROM failed_webhook_jobs WHERE failed_at > NOW() - INTERVAL '24 hours';" 2>&1
|
||||
echo "--- scheduler_heartbeats with failures ---"
|
||||
sudo -u postgres psql -d liderra -tA -c "SELECT command_name, consecutive_failures, last_run_at FROM scheduler_heartbeats WHERE consecutive_failures > 0 ORDER BY consecutive_failures DESC;" 2>&1
|
||||
|
||||
echo
|
||||
echo "=== DONE ==="
|
||||
REMOTE
|
||||
|
||||
- name: Print summary
|
||||
if: always() && steps.window.outputs.skip != 'true'
|
||||
run: |
|
||||
{
|
||||
echo "## Stage 5 daily monitor — $(date -u +%Y-%m-%d)"
|
||||
echo
|
||||
echo '```'
|
||||
cat /tmp/monitor.log 2>/dev/null || echo "(no output)"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload as artifact
|
||||
if: always() && steps.window.outputs.skip != 'true'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: monitor-${{ github.run_id }}
|
||||
path: /tmp/monitor.log
|
||||
retention-days: 14
|
||||
|
||||
- name: Cleanup SSH key
|
||||
if: always()
|
||||
run: rm -f ~/.ssh/liderra_deploy
|
||||
Reference in New Issue
Block a user