backup.yml

name: Scheduled Backup

on:
  schedule:
    - cron: '0 3 * * *'  # Daily at 3 AM UTC
  workflow_dispatch:

env:
  PROJECT_DIR: /home/vasyl/projects/onlinelib/textstack
  BACKUP_DIR: /home/vasyl/backups/textstack
  # Opt JS actions into Node 24 ahead of June 2026 forced cutover.
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true'

# No checkout and no API calls: the job token gets nothing.
permissions: {}

jobs:
  backup:
    runs-on: self-hosted
    # 13m typical.
    timeout-minutes: 45

    steps:
      - name: Set backup date
        run: |
          echo "BACKUP_DATE=$(date +%F)" >> $GITHUB_ENV
          # The V2 database (ADR-030): same container, its own dump, files prefixed by its name.
          v2=$(sed -n 's/^V2_POSTGRES_DB=//p' $PROJECT_DIR/.env | tr -d "\"' ")
          echo "V2_DB=${v2:-textstack_v2}" >> $GITHUB_ENV

      - name: Create backup directory
        run: mkdir -p $BACKUP_DIR

      - name: Backup database
        run: |
          # Without pipefail a failed pg_dump still exits 0 through gzip and leaves
          # an empty archive that every later step accepts.
          set -o pipefail
          docker exec textstack_db_prod pg_dump -U textstack_prod textstack_prod | gzip > $BACKUP_DIR/db-$BACKUP_DATE.sql.gz
          # pg_dump writes this trailer last; a truncated dump doesn't have it.
          zcat $BACKUP_DIR/db-$BACKUP_DATE.sql.gz | tail -n 5 | grep -q 'PostgreSQL database dump complete' \
            || { echo "::error::db-$BACKUP_DATE.sql.gz has no dump-complete trailer"; exit 1; }
          echo "Database backup: db-$BACKUP_DATE.sql.gz"

      # V2 next to legacy (#1043). Skipped, green, while the database does not exist: migrator-v2 is
      # behind a profile, so a fresh server has none. continue-on-error: a failure here must not stop
      # one legacy step below; the last step turns it red. Means nothing for the site.
      # The script writes the file only once complete: the R2 step sends it only if it exists.
      - name: Backup V2 database
        id: v2
        continue-on-error: true
        run: |
          POSTGRES_USER=textstack_prod bash $PROJECT_DIR/infra/scripts/pg-dump-checked.sh \
            $V2_DB $BACKUP_DIR/$V2_DB-db-$BACKUP_DATE.sql.gz

      - name: Backup storage
        run: |
          # GNU tar exits 1 when a file changed while being read (readers upload while we
          # archive) — the archive is still complete for everything else. Only >1 is fatal.
          tar czf $BACKUP_DIR/storage-$BACKUP_DATE.tar.gz -C $PROJECT_DIR/data storage || [ $? -eq 1 ]
          echo "Storage backup: storage-$BACKUP_DATE.tar.gz"

      - name: Cleanup old backups (keep 2 newest of each type — history lives in R2)
        run: |
          cd $BACKUP_DIR
          ls -1t db-*.sql.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
          ls -1t storage-*.tar.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
          ls -1t pre-deploy-*.sql.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
          ls -1t $V2_DB-db-*.sql.gz 2>/dev/null | tail -n +3 | xargs -r rm -v
          ls -1t $V2_DB-pre-deploy-*.sql.gz 2>/dev/null | tail -n +3 | xargs -r rm -v

      - name: Verify archive integrity
        run: |
          ls -lh $BACKUP_DIR/*.gz | tail -5
          gunzip -t $BACKUP_DIR/db-$BACKUP_DATE.sql.gz

      - name: Verify backup restores into fresh postgres
        run: |
          cd $PROJECT_DIR
          ./infra/scripts/backup-verify.sh $BACKUP_DIR/db-$BACKUP_DATE.sql.gz

      # Off-site copy. Everything above lives on the same box as the data it protects; this is the
      # only copy that survives losing the server. restic (in Docker, nothing installed on the host)
      # to Cloudflare R2: encrypted client-side, deduplicated, so the storage folder costs its size
      # once, not once per night. Keeps 7 nightly + 3 monthly snapshots.
      # .env goes too — without it a restored box cannot start.
      # RESTIC_PASSWORD must also live in the owner's password manager: without it the copy is noise.
      # Skipped (with a warning) until the secrets exist, so the job stays green before setup.
      - name: Off-site backup to R2 (restic)
        env:
          RESTIC_REPOSITORY: ${{ secrets.RESTIC_REPOSITORY }}
          RESTIC_PASSWORD: ${{ secrets.RESTIC_PASSWORD }}
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
        run: |
          set -euo pipefail
          if [ -z "$RESTIC_REPOSITORY" ]; then
            echo "::warning::RESTIC_REPOSITORY secret not set — off-site backup skipped"
            exit 0
          fi
          restic() {
            docker run --rm -i \
              -e RESTIC_REPOSITORY -e RESTIC_PASSWORD -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
              -v "$HOME/.cache/restic:/root/.cache/restic" \
              -v "$PROJECT_DIR/data/storage:/backup/storage:ro" \
              -v "$PROJECT_DIR/.env:/backup/env:ro" \
              restic/restic:0.17.3@sha256:8f5a62b422a2cb1277ea0dd6e826fe1acf649e5b9f02d60e5268d5fd1976255a "$@"
          }
          mkdir -p "$HOME/.cache/restic"
          # Guard: stay inside R2's free 10 GB. Over 9 GB the job fails (email) instead of quietly
          # costing money. Dedup means retention barely matters — size ≈ live data (~3-4 GB, 2026-10).
          LIMIT=$((9 * 1024 * 1024 * 1024))
          repo_size() { restic stats --mode raw-data --json | grep -o '"total_size":[0-9]*' | cut -d: -f2; }
          restic cat config >/dev/null 2>&1 || restic init
          size=$(repo_size)
          if [ "$size" -gt "$LIMIT" ]; then
            echo "::error::R2 backup repo is $((size / 1024 / 1024)) MB (> 9 GB) — not adding a snapshot"
            exit 1
          fi
          restic backup --host textstack-prod --tag nightly /backup
          # The dump goes in UNcompressed, through stdin. A .gz differs byte-for-byte every night,
          # so restic stored all ~1.2 GB of it each time (2026-10-05). Plain SQL dedups by chunk:
          # a night adds only what changed; restic compresses it itself.
          zcat "$BACKUP_DIR/db-$BACKUP_DATE.sql.gz" \
            | restic backup --host textstack-prod --tag nightly --stdin --stdin-filename db.sql
          # V2 before forget/check and the size guard, so it is pruned, integrity-checked and counted
          # against 9 GB like the rest. Its failure is recorded, not fatal here: legacy's forget/check
          # still run, and the last step turns the job red. Its own stdin path: restore-drill.yml reads
          # "no /db_v2.sql snapshot" as "no V2 in this backup".
          v2="$BACKUP_DIR/$V2_DB-db-$BACKUP_DATE.sql.gz"
          if [ -f "$v2" ]; then
            zcat "$v2" | restic backup --host textstack-prod --tag nightly --stdin --stdin-filename db_v2.sql \
              || { echo "::error::V2 upload to R2 failed"; echo "V2_R2_FAILED=1" >> "$GITHUB_ENV"; }
          else
            echo "No V2 dump tonight: nothing V2 sent to R2"
          fi
          restic forget --host textstack-prod --keep-daily 7 --keep-monthly 3 --prune
          restic check
          size=$(repo_size)
          echo "R2 backup repo: $((size / 1024 / 1024)) MB"
          if [ "$size" -gt "$LIMIT" ]; then
            echo "::error::R2 backup repo is $((size / 1024 / 1024)) MB (> 9 GB)"
            exit 1
          fi

      # No nightly SSG rebuild since #992: the SSG tree is frozen (search sent 6 clicks in 90 days).
      # Rebuild by hand when the catalog changes: admin panel or `make rebuild-ssg`.

      # Runs even when the backup failed — a full disk is the usual reason it does.
      # Failing here fires the same failure notification as a broken backup.
      - name: Disk space alarm (>= 85%)
        if: always()
        run: |
          bad=0
          for mnt in / "$(docker info -f '{{.DockerRootDir}}')"; do
            pct=$(df --output=pcent "$mnt" | tail -1 | tr -dc '0-9')
            echo "$mnt: ${pct}% used"
            [ "$pct" -lt 85 ] || { echo "::error::$mnt is ${pct}% full"; bad=1; }
          done
          exit $bad

      # The V2 dump above is continue-on-error and its R2 upload only records a failure, so every
      # legacy step runs; this makes either failure the job's red (and email) anyway.
      - name: V2 backup failed
        if: always() && (steps.v2.outcome == 'failure' || env.V2_R2_FAILED == '1')
        run: |
          echo "::error::The $V2_DB dump or its R2 upload failed (see 'Backup V2 database' / 'Off-site backup'). Legacy backup unaffected."
          exit 1