Дмитрий
ef19b9f256
ремонт: prev rebuild left 6 mismatches на activity_log_y2026_m05 Previous workflow used t::text::bytea (full row). Canonical algorithm uses explicit ROW(col1, ..., NULL::bytea, ..., coln)::text::bytea with COLUMN_CONFIG. Workflow now switches ROW expression by partition family. +6 cspell words: psql/euo/coln/esac/cnt/bytea.
222 lines
8.5 KiB
YAML
222 lines
8.5 KiB
YAML
name: Rebuild audit hash chain via postgres superuser (F1 cleanup)
|
||
|
||
# Closes deferred F1 item from docs/incidents/2026-05-29-disk-full-pg-recovery.md §4.1.
|
||
# Sequential hash recomputation в plpgsql DO-блоке через sudo -u postgres psql.
|
||
# Identical алгоритм с trigger audit_chain_hash() (post-F1 advisory-lock version),
|
||
# но применённый к existing rows.
|
||
#
|
||
# Использование:
|
||
# gh workflow run f1-rebuild-via-superuser.yml \
|
||
# -f partition=activity_log_y2026_m05 -f from_id=599 -f confirm_apply=true
|
||
#
|
||
# Safety:
|
||
# - Partition name whitelist (только заранее известные сломанные партиции).
|
||
# - dry_run=true mode показывает count + anchor prev_hash без UPDATE.
|
||
# - Trigger audit_chain_hash отключён через SET LOCAL session_replication_role=replica
|
||
# (постоянный disable невозможен — после COMMIT триггер опять активен).
|
||
# - audit_block_mutation также подавлен через session_replication_role=replica.
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
inputs:
|
||
partition:
|
||
description: 'Partition name (whitelist: activity_log_y2026_m05, balance_transactions_y2026_m05)'
|
||
required: true
|
||
type: string
|
||
from_id:
|
||
description: 'First broken id (rebuild from here onward)'
|
||
required: true
|
||
type: string
|
||
dry_run:
|
||
description: 'Dry-run (показать count + anchor без UPDATE)'
|
||
required: false
|
||
default: 'false'
|
||
type: boolean
|
||
confirm_apply:
|
||
description: 'Подтверждаю rebuild на проде (требуется если dry_run=false)'
|
||
required: false
|
||
default: 'false'
|
||
type: boolean
|
||
|
||
jobs:
|
||
rebuild:
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 15
|
||
|
||
env:
|
||
LIDERRA_HOST: 111.88.246.137
|
||
LIDERRA_USER: ubuntu
|
||
PARTITION: ${{ github.event.inputs.partition }}
|
||
FROM_ID: ${{ github.event.inputs.from_id }}
|
||
DRY_RUN: ${{ github.event.inputs.dry_run }}
|
||
CONFIRM: ${{ github.event.inputs.confirm_apply }}
|
||
|
||
steps:
|
||
- name: Validate inputs
|
||
run: |
|
||
set -euo pipefail
|
||
|
||
# Whitelist partition names (защита от arbitrary table names)
|
||
ALLOWED='^(activity_log_y2026_m05|balance_transactions_y2026_m05)$'
|
||
if ! [[ "$PARTITION" =~ $ALLOWED ]]; then
|
||
echo "::error::partition '$PARTITION' not in whitelist: $ALLOWED"
|
||
exit 1
|
||
fi
|
||
|
||
# from_id is positive integer
|
||
if ! [[ "$FROM_ID" =~ ^[0-9]+$ ]]; then
|
||
echo "::error::from_id must be positive integer, got '$FROM_ID'"
|
||
exit 1
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" != "true" && "$CONFIRM" != "true" ]]; then
|
||
echo "::error::Either dry_run=true OR confirm_apply=true must be set"
|
||
exit 1
|
||
fi
|
||
|
||
echo "Inputs OK: partition=$PARTITION, from_id=$FROM_ID, dry_run=$DRY_RUN, confirm_apply=$CONFIRM"
|
||
|
||
- name: Setup SSH key
|
||
run: |
|
||
mkdir -p ~/.ssh
|
||
echo "${{ secrets.LIDERRA_SSH_KEY }}" > ~/.ssh/liderra_deploy
|
||
chmod 600 ~/.ssh/liderra_deploy
|
||
ssh-keyscan -H ${{ env.LIDERRA_HOST }} >> ~/.ssh/known_hosts 2>/dev/null
|
||
|
||
- name: Run rebuild on prod
|
||
run: |
|
||
ssh -i ~/.ssh/liderra_deploy ${{ env.LIDERRA_USER }}@${{ env.LIDERRA_HOST }} \
|
||
"PARTITION='$PARTITION' FROM_ID='$FROM_ID' DRY_RUN='$DRY_RUN' bash -s" <<'REMOTE' | tee /tmp/f1-rebuild.log
|
||
set +e
|
||
|
||
echo "=== 1. Anchor + count preview ==="
|
||
sudo -u postgres psql -d liderra -v ON_ERROR_STOP=1 <<SQL
|
||
\set partition $PARTITION
|
||
\set from_id $FROM_ID
|
||
|
||
-- Anchor: log_hash of row right BEFORE from_id (=> prev_hash for from_id)
|
||
SELECT
|
||
(SELECT id FROM :"partition" WHERE id < :from_id ORDER BY id DESC LIMIT 1) AS anchor_id,
|
||
encode((SELECT log_hash FROM :"partition" WHERE id < :from_id ORDER BY id DESC LIMIT 1), 'hex') AS anchor_log_hash,
|
||
(SELECT COUNT(*) FROM :"partition" WHERE id >= :from_id) AS rows_to_rebuild,
|
||
(SELECT MIN(id) FROM :"partition" WHERE id >= :from_id) AS first_id,
|
||
(SELECT MAX(id) FROM :"partition" WHERE id >= :from_id) AS last_id;
|
||
SQL
|
||
PRE_RC=$?
|
||
if [[ $PRE_RC -ne 0 ]]; then
|
||
echo "::error::Pre-check failed (RC=$PRE_RC)"
|
||
exit $PRE_RC
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == "true" ]]; then
|
||
echo
|
||
echo "=== DRY RUN — no changes applied ==="
|
||
exit 0
|
||
fi
|
||
|
||
echo
|
||
echo "=== 2. APPLY: rebuild hash chain on $PARTITION from id=$FROM_ID ==="
|
||
# Canonical algorithm (mirrors app/app/Console/Commands/AuditRebuildChain.php):
|
||
# builds explicit ROW(col1, col2, ..., NULL::bytea on log_hash position, ..., coln)::text::bytea
|
||
# so hash matches what audit:verify-chains computes (which uses same COLUMN_CONFIG).
|
||
case "$PARTITION" in
|
||
activity_log_*)
|
||
ROW_EXPR="ROW(t.id, t.tenant_id, t.user_id, t.deal_id, t.event, t.old_value, t.new_value, t.context, t.ip_address, t.user_agent, NULL::bytea, t.created_at)"
|
||
;;
|
||
balance_transactions_*)
|
||
ROW_EXPR="ROW(t.id, t.tenant_id, t.type, t.amount_rub, t.amount_leads, t.balance_rub_after, t.balance_leads_after, t.description, t.related_type, t.related_id, t.user_id, t.admin_user_id, NULL::bytea, t.created_at)"
|
||
;;
|
||
*)
|
||
echo "::error::Unknown partition family — add ROW_EXPR mapping"
|
||
exit 1
|
||
;;
|
||
esac
|
||
echo "Using ROW expression: $ROW_EXPR"
|
||
|
||
sudo -u postgres psql -d liderra -v ON_ERROR_STOP=1 <<SQL
|
||
BEGIN;
|
||
SET LOCAL session_replication_role = 'replica';
|
||
|
||
DO \$rebuild\$
|
||
DECLARE
|
||
cur_id BIGINT;
|
||
prev_hash BYTEA;
|
||
new_hash BYTEA;
|
||
cnt INTEGER := 0;
|
||
partition_name TEXT := '$PARTITION';
|
||
start_id BIGINT := $FROM_ID;
|
||
row_expr TEXT := '$ROW_EXPR';
|
||
BEGIN
|
||
EXECUTE format(
|
||
'SELECT log_hash FROM %I WHERE id < \$1 ORDER BY id DESC LIMIT 1',
|
||
partition_name
|
||
)
|
||
INTO prev_hash
|
||
USING start_id;
|
||
|
||
RAISE NOTICE 'Anchor prev_hash: %', COALESCE(encode(prev_hash, 'hex'), '<NULL — start of chain>');
|
||
|
||
FOR cur_id IN
|
||
EXECUTE format(
|
||
'SELECT id FROM %I WHERE id >= \$1 ORDER BY id',
|
||
partition_name
|
||
)
|
||
USING start_id
|
||
LOOP
|
||
-- Compute new_hash with explicit ROW(...) expression (canonical, matches verify-chains)
|
||
EXECUTE format(
|
||
'SELECT digest(COALESCE(\$1, ''''::bytea) || %s::text::bytea, ''sha256'') FROM %I t WHERE id = \$2',
|
||
row_expr, partition_name
|
||
)
|
||
INTO new_hash
|
||
USING prev_hash, cur_id;
|
||
|
||
EXECUTE format('UPDATE %I SET log_hash = \$1 WHERE id = \$2', partition_name)
|
||
USING new_hash, cur_id;
|
||
|
||
prev_hash := new_hash;
|
||
cnt := cnt + 1;
|
||
END LOOP;
|
||
|
||
RAISE NOTICE 'Rebuilt % rows. Last log_hash: %', cnt, encode(prev_hash, 'hex');
|
||
END
|
||
\$rebuild\$;
|
||
|
||
COMMIT;
|
||
SQL
|
||
APPLY_RC=$?
|
||
|
||
echo
|
||
echo "=== 3. Verify: no NULL log_hash в обновлённых строках ==="
|
||
sudo -u postgres psql -d liderra <<SQL
|
||
\set partition $PARTITION
|
||
\set from_id $FROM_ID
|
||
SELECT
|
||
COUNT(*) FILTER (WHERE log_hash IS NULL) AS null_count,
|
||
COUNT(*) AS total,
|
||
MIN(id) AS first_id,
|
||
MAX(id) AS last_id
|
||
FROM :"partition"
|
||
WHERE id >= :from_id;
|
||
SQL
|
||
|
||
echo
|
||
echo "=== Apply RC: $APPLY_RC ==="
|
||
exit $APPLY_RC
|
||
REMOTE
|
||
|
||
- name: Print summary
|
||
if: always()
|
||
run: |
|
||
{
|
||
echo "## F1 chain rebuild — $PARTITION (from_id=$FROM_ID, dry_run=$DRY_RUN)"
|
||
echo
|
||
echo '```'
|
||
cat /tmp/f1-rebuild.log 2>/dev/null || echo "(no log)"
|
||
echo '```'
|
||
} >> "$GITHUB_STEP_SUMMARY"
|
||
|
||
- name: Cleanup SSH key
|
||
if: always()
|
||
run: rm -f ~/.ssh/liderra_deploy
|