34bcc570ad
ремонт: logrotate отказал rotation PG log из-за insecure parent dir permissions /var/log/postgresql/ имеет permissions drwxrwxr-t (group-writable + sticky). Logrotate refuses to rotate без явного su directive в config. Стандарт postgresql-common тоже использует 'su' — копирую идиому.
168 lines
5.8 KiB
YAML
168 lines
5.8 KiB
YAML
name: Setup logrotate for Laravel logs (incident prevention)
|
||
|
||
# Incident response prevention: 8.7G laravel.log заполнил диск (29.05.2026).
|
||
# Существующий daily rotation (laravel.log.1) недостаточен — за один день шторма
|
||
# accumulated 8.7G. Нужна size-based rotation с лимитом.
|
||
#
|
||
# This workflow installs /etc/logrotate.d/laravel-liderra config:
|
||
# - size 50M (rotate when file >= 50MB, не daily)
|
||
# - rotate 5 (keep 5 rotated copies)
|
||
# - compress (gzip rotated files)
|
||
# - copytruncate (atomic copy + truncate inode-preserving, Laravel handle continues)
|
||
# - notifempty (skip if empty)
|
||
# - su www-data www-data (correct ownership)
|
||
#
|
||
# Тестируется logrotate --debug сразу после установки.
|
||
#
|
||
# Ref: root-cause analysis incident 2026-05-29
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
inputs:
|
||
confirm_apply:
|
||
description: 'Подтверждаю установку logrotate конфига на проде'
|
||
required: true
|
||
default: 'false'
|
||
type: boolean
|
||
|
||
jobs:
|
||
setup:
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 5
|
||
|
||
env:
|
||
LIDERRA_HOST: 111.88.246.137
|
||
LIDERRA_USER: ubuntu
|
||
CONFIRM: ${{ github.event.inputs.confirm_apply }}
|
||
|
||
steps:
|
||
- name: Guard
|
||
run: |
|
||
if [[ "$CONFIRM" != "true" ]]; then
|
||
echo "::error::confirm_apply=true required"
|
||
exit 1
|
||
fi
|
||
|
||
- name: Setup SSH key
|
||
run: |
|
||
mkdir -p ~/.ssh
|
||
echo "${{ secrets.LIDERRA_SSH_KEY }}" > ~/.ssh/liderra_deploy
|
||
chmod 600 ~/.ssh/liderra_deploy
|
||
ssh-keyscan -H ${{ env.LIDERRA_HOST }} >> ~/.ssh/known_hosts 2>/dev/null
|
||
|
||
- name: Install logrotate config + verify
|
||
run: |
|
||
ssh -i ~/.ssh/liderra_deploy ${{ env.LIDERRA_USER }}@${{ env.LIDERRA_HOST }} \
|
||
"bash -s" <<'REMOTE' | tee /tmp/logrotate-setup.log
|
||
set +e
|
||
|
||
echo "=== 1. Discover Laravel logs path ==="
|
||
LARAVEL_LOG_DIR=""
|
||
for candidate in /var/www/liderra/app/storage/logs /var/www/lidpotok/storage/logs; do
|
||
if [[ -d "$candidate" ]]; then
|
||
LARAVEL_LOG_DIR="$candidate"
|
||
break
|
||
fi
|
||
done
|
||
echo "LARAVEL_LOG_DIR=$LARAVEL_LOG_DIR"
|
||
if [[ -z "$LARAVEL_LOG_DIR" ]]; then
|
||
echo "::error::Cannot find Laravel logs directory"
|
||
exit 1
|
||
fi
|
||
echo "Current sizes:"
|
||
sudo du -sh "$LARAVEL_LOG_DIR"/*.log 2>/dev/null | head -10
|
||
|
||
echo
|
||
echo "=== 2. Write logrotate config to /etc/logrotate.d/laravel-liderra ==="
|
||
sudo tee /etc/logrotate.d/laravel-liderra > /dev/null <<EOF
|
||
$LARAVEL_LOG_DIR/*.log {
|
||
size 50M
|
||
rotate 5
|
||
compress
|
||
delaycompress
|
||
missingok
|
||
notifempty
|
||
copytruncate
|
||
su www-data www-data
|
||
create 0644 www-data www-data
|
||
}
|
||
EOF
|
||
echo "Wrote config:"
|
||
sudo cat /etc/logrotate.d/laravel-liderra
|
||
sudo chmod 0644 /etc/logrotate.d/laravel-liderra
|
||
|
||
echo
|
||
echo "=== 3. Verify config syntax via logrotate --debug ==="
|
||
sudo logrotate --debug /etc/logrotate.d/laravel-liderra 2>&1 | head -30
|
||
|
||
echo
|
||
echo "=== 4. Trigger rotation now (--force) for clean state ==="
|
||
sudo logrotate --force /etc/logrotate.d/laravel-liderra 2>&1 | tail -10
|
||
|
||
echo
|
||
echo "=== 5. PostgreSQL log rotation config ==="
|
||
# Default Ubuntu postgresql-common rotates daily without size cap.
|
||
# We override with size 100M / rotate 7 / postrotate SIGHUP (PG reopens log).
|
||
# Higher alpha order than postgresql-common → processed later → wins on same files.
|
||
sudo tee /etc/logrotate.d/postgresql-liderra > /dev/null <<EOF
|
||
/var/log/postgresql/*.log {
|
||
su postgres postgres
|
||
size 100M
|
||
rotate 7
|
||
compress
|
||
delaycompress
|
||
missingok
|
||
notifempty
|
||
create 0640 postgres adm
|
||
sharedscripts
|
||
postrotate
|
||
# SIGHUP postmaster для re-open log file (standard PG idiom).
|
||
# PG holds log file handle open — без SIGHUP write goes to old (deleted) inode.
|
||
if [ -f /var/run/postgresql/16-main.pid ]; then
|
||
kill -HUP \$(cat /var/run/postgresql/16-main.pid) 2>/dev/null || true
|
||
fi
|
||
endscript
|
||
}
|
||
EOF
|
||
echo "Wrote /etc/logrotate.d/postgresql-liderra:"
|
||
sudo cat /etc/logrotate.d/postgresql-liderra
|
||
sudo chmod 0644 /etc/logrotate.d/postgresql-liderra
|
||
|
||
echo
|
||
echo "=== 6. Verify PG logrotate syntax ==="
|
||
sudo logrotate --debug /etc/logrotate.d/postgresql-liderra 2>&1 | head -20
|
||
|
||
echo
|
||
echo "=== 7. Force PG log rotation now (clean state) ==="
|
||
sudo logrotate --force /etc/logrotate.d/postgresql-liderra 2>&1 | tail -10
|
||
|
||
echo
|
||
echo "=== 8. AFTER: PG log directory state ==="
|
||
sudo ls -lah /var/log/postgresql/ 2>&1 | head -10
|
||
|
||
echo
|
||
echo "=== 9. AFTER: Laravel log directory state ==="
|
||
sudo ls -lah "$LARAVEL_LOG_DIR/" 2>&1 | head -20
|
||
echo
|
||
echo "=== 10. Disk free ==="
|
||
df -h / 2>&1 | head -3
|
||
|
||
echo
|
||
echo "=== DONE ==="
|
||
REMOTE
|
||
|
||
- name: Print summary
|
||
if: always()
|
||
run: |
|
||
{
|
||
echo "## logrotate setup"
|
||
echo
|
||
echo '```'
|
||
cat /tmp/logrotate-setup.log 2>/dev/null || echo "(no log)"
|
||
echo '```'
|
||
} >> "$GITHUB_STEP_SUMMARY"
|
||
|
||
- name: Cleanup SSH key
|
||
if: always()
|
||
run: rm -f ~/.ssh/liderra_deploy
|