Repository navigation
deploy-docker-image #132
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Docker Image on Self-Hosted | |
| on: | |
| repository_dispatch: | |
| types: | |
| - deploy-docker-image | |
| workflow_dispatch: | |
| inputs: | |
| image_tag: | |
| description: "Immutable image tag to deploy (e.g. sha-abc1234). 'latest' is rejected - it can't be verified by /version and is not a safe rollback target." | |
| required: true | |
| concurrency: | |
| group: deploy-self-hosted | |
| cancel-in-progress: false | |
| env: | |
| IMAGE: ghcr.io/butr/crashreportserver | |
| HEALTH_URL: https://crash.butr.link/healthz | |
| VERSION_URL: https://crash.butr.link/version | |
| STACK: ${{ vars.DEPLOY_STACK }} | |
| SERVICE: ${{ vars.DEPLOY_SERVICE }} | |
| COMPOSE: ${{ vars.DEPLOY_COMPOSE }} | |
| IMAGE_TAG: ${{ github.event.client_payload.image_tag || inputs.image_tag }} | |
| STATE_DIR: /tmp/crashreportserver-deploy | |
| jobs: | |
| deploy: | |
| name: Deploy | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: "self-hosted-backend" | |
| url: "https://crash.butr.link" | |
| steps: | |
| - name: Mask deploy topology in logs | |
| timeout-minutes: 1 | |
| run: | | |
| echo "::add-mask::${{ vars.DEPLOY_STACK }}" | |
| echo "::add-mask::${{ vars.DEPLOY_SERVICE }}" | |
| echo "::add-mask::${{ vars.DEPLOY_COMPOSE }}" | |
| - name: Validate image tag | |
| timeout-minutes: 1 | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${IMAGE_TAG}" ] || [ "${IMAGE_TAG}" = "latest" ]; then | |
| echo "::error::IMAGE_TAG must be an immutable tag (e.g. sha-abc1234); got '${IMAGE_TAG:-<empty>}'. 'latest' is rejected because /version cannot confirm it and it is not a safe rollback target." | |
| exit 1 | |
| fi | |
| echo "Deploying tag: ${IMAGE_TAG}" | |
| - name: Pull image and capture rollback state | |
| timeout-minutes: 10 | |
| uses: appleboy/ssh-action@v1 | |
| with: | |
| host: ${{ secrets.HOST }} | |
| username: ${{ secrets.USERNAME }} | |
| password: ${{ secrets.PASSWORD }} | |
| port: ${{ secrets.PORT }} | |
| envs: IMAGE,SERVICE,COMPOSE,IMAGE_TAG,STATE_DIR | |
| script: | | |
| set -eu | |
| # Start from a clean slate so a previous deploy's stale state can never be read as this deploy's. | |
| rm -rf "${STATE_DIR}"; mkdir -p "${STATE_DIR}" | |
| echo "Deploying ${IMAGE}:${IMAGE_TAG}" | |
| docker pull "${IMAGE}:${IMAGE_TAG}" | |
| # Capture the EXACT currently-running image WITH its @sha256 digest, plus the replica count, as the | |
| # rollback target. Pinning by digest makes rollback immune to tag movement - 'latest' (and even this | |
| # commit's sha tag) may already point at the very build we're about to deploy. | |
| OLD_IMAGE_REF="$(docker service inspect --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' "${SERVICE}" 2>/dev/null || true)" | |
| OLD_REPLICAS="$(docker service inspect --format '{{.Spec.Mode.Replicated.Replicas}}' "${SERVICE}" 2>/dev/null || true)" | |
| if [ -n "${OLD_IMAGE_REF}" ]; then | |
| echo "${OLD_IMAGE_REF}" > "${STATE_DIR}/old_image_ref" | |
| echo "${OLD_REPLICAS:-1}" > "${STATE_DIR}/old_replicas" | |
| # Tag (no repo, no digest) is only used by the prune step's keep-list. | |
| OLD_TAG="${OLD_IMAGE_REF%@*}"; OLD_TAG="${OLD_TAG##*:}" | |
| echo "${OLD_TAG}" > "${STATE_DIR}/old_tag" | |
| echo "Previous image: ${OLD_IMAGE_REF} (replicas ${OLD_REPLICAS:-1})" | |
| else | |
| echo "::warning::No running service found; automatic image rollback will be unavailable" | |
| fi | |
| # Last-applied migration BEFORE migrating = the rollback target. Read with the NEW image. | |
| # Extract only the marker line so interleaved logging can't be mistaken for the id. | |
| DB_ENV="$(dirname "${COMPOSE}")/db.env" | |
| MIGRATION_OUTPUT="$(docker run --rm --env-file "${DB_ENV}" "${IMAGE}:${IMAGE_TAG}" current-migration)" | |
| PREV_MIGRATION="$(printf '%s\n' "${MIGRATION_OUTPUT}" | sed -n 's/^::MIGRATION_ID:://p' | tail -n1)" | |
| echo "${PREV_MIGRATION}" > "${STATE_DIR}/prev_migration" | |
| echo "Rollback migration target: ${PREV_MIGRATION:-<none>}" | |
| - name: Stop service | |
| timeout-minutes: 3 | |
| uses: appleboy/ssh-action@v1 | |
| with: | |
| host: ${{ secrets.HOST }} | |
| username: ${{ secrets.USERNAME }} | |
| password: ${{ secrets.PASSWORD }} | |
| port: ${{ secrets.PORT }} | |
| envs: SERVICE,STATE_DIR | |
| script: | | |
| set -eu | |
| docker service scale "${SERVICE}=0" | |
| # Marker: from here on the running deployment has been disrupted, so rollback is warranted on failure. | |
| # If a step before this fails, the service is still healthy and rollback must NOT touch it. | |
| touch "${STATE_DIR}/service_stopped" | |
| - name: Apply migrations | |
| timeout-minutes: 18 | |
| uses: appleboy/ssh-action@v1 | |
| with: | |
| host: ${{ secrets.HOST }} | |
| username: ${{ secrets.USERNAME }} | |
| password: ${{ secrets.PASSWORD }} | |
| port: ${{ secrets.PORT }} | |
| envs: IMAGE,COMPOSE,IMAGE_TAG | |
| command_timeout: 15m | |
| script: | | |
| set -eu | |
| docker run --rm --env-file "$(dirname "${COMPOSE}")/db.env" "${IMAGE}:${IMAGE_TAG}" migrate | |
| - name: Deploy new tag | |
| timeout-minutes: 5 | |
| uses: appleboy/ssh-action@v1 | |
| with: | |
| host: ${{ secrets.HOST }} | |
| username: ${{ secrets.USERNAME }} | |
| password: ${{ secrets.PASSWORD }} | |
| port: ${{ secrets.PORT }} | |
| envs: STACK,COMPOSE,IMAGE_TAG | |
| script: | | |
| set -eu | |
| IMAGE_TAG="${IMAGE_TAG}" docker stack deploy \ | |
| --resolve-image always --prune \ | |
| -c "${COMPOSE}" "${STACK}" | |
| - name: Health check | |
| timeout-minutes: 5 | |
| run: | | |
| set -euo pipefail | |
| for i in $(seq 1 30); do | |
| if curl -fsS --max-time 5 "${HEALTH_URL}" >/dev/null; then | |
| echo "Healthy after ${i} check(s)"; exit 0 | |
| fi | |
| echo "Health check ${i}/30 not ready; waiting 5s" | |
| sleep 5 | |
| done | |
| echo "::error::Service did not become healthy" | |
| exit 1 | |
| - name: Verify deployed version | |
| timeout-minutes: 5 | |
| run: | | |
| set -euo pipefail | |
| for i in $(seq 1 30); do | |
| LIVE="$(curl -fsS --max-time 5 "${VERSION_URL}" || true)" | |
| echo "Attempt ${i}/30: /version -> '${LIVE:-<unreachable>}' (expected '${IMAGE_TAG}')" | |
| if [ "${LIVE}" = "${IMAGE_TAG}" ]; then | |
| echo "Confirmed live version: ${LIVE}"; exit 0 | |
| fi | |
| sleep 5 | |
| done | |
| echo "::error::Live /version never matched '${IMAGE_TAG}'" | |
| exit 1 | |
| - name: Prune old service images | |
| if: success() | |
| timeout-minutes: 5 | |
| uses: appleboy/ssh-action@v1 | |
| with: | |
| host: ${{ secrets.HOST }} | |
| username: ${{ secrets.USERNAME }} | |
| password: ${{ secrets.PASSWORD }} | |
| port: ${{ secrets.PORT }} | |
| envs: IMAGE,IMAGE_TAG,STATE_DIR | |
| script: | | |
| set -eu | |
| OLD_TAG="$(cat "${STATE_DIR}/old_tag" 2>/dev/null || true)" | |
| KEEP="${IMAGE_TAG} latest ${OLD_TAG}" | |
| echo "Keeping tags: ${KEEP}" | |
| docker images --format '{{.Repository}}:{{.Tag}}' \ | |
| | awk -v repo="${IMAGE}:" 'index($0, repo) == 1' \ | |
| | while read -r ref; do | |
| tag="${ref##*:}" | |
| keep=0 | |
| for k in ${KEEP}; do [ "${tag}" = "${k}" ] && keep=1 && break; done | |
| if [ "${keep}" = "1" ]; then | |
| echo "keep ${ref}" | |
| else | |
| echo "remove ${ref}" | |
| docker rmi "${ref}" || echo "::warning::could not remove ${ref} (still in use?)" | |
| fi | |
| done | |
| # Mop up layers left dangling by the removed/replaced tags. | |
| docker image prune -f >/dev/null 2>&1 || true | |
| - name: Rollback on failure | |
| if: failure() | |
| timeout-minutes: 20 | |
| uses: appleboy/ssh-action@v1 | |
| with: | |
| host: ${{ secrets.HOST }} | |
| username: ${{ secrets.USERNAME }} | |
| password: ${{ secrets.PASSWORD }} | |
| port: ${{ secrets.PORT }} | |
| envs: IMAGE,SERVICE,COMPOSE,IMAGE_TAG,STATE_DIR,HEALTH_URL,VERSION_URL | |
| command_timeout: 15m | |
| script: | | |
| set -eu | |
| # Only roll back if we actually disrupted the deployment. If a step before "Stop service" failed, | |
| # the original service is still running and healthy - touching it would cause a needless outage. | |
| if [ ! -f "${STATE_DIR}/service_stopped" ]; then | |
| echo "Service was never stopped; deployment unchanged. Skipping rollback." | |
| exit 0 | |
| fi | |
| OLD_IMAGE_REF="$(cat "${STATE_DIR}/old_image_ref" 2>/dev/null || true)" | |
| OLD_REPLICAS="$(cat "${STATE_DIR}/old_replicas" 2>/dev/null || true)" | |
| PREV_MIGRATION="$(cat "${STATE_DIR}/prev_migration" 2>/dev/null || true)" | |
| echo "::error::Deploy of ${IMAGE}:${IMAGE_TAG} failed - rolling back to ${OLD_IMAGE_REF:-<unknown>} (migration target ${PREV_MIGRATION:-none})" | |
| # Take the broken app fully down before reverting the schema (migrate-down with the NEW image, since | |
| # only it defines the new Down()). | |
| docker service scale "${SERVICE}=0" | |
| if [ -n "${PREV_MIGRATION}" ]; then | |
| if ! docker run --rm --env-file "$(dirname "${COMPOSE}")/db.env" "${IMAGE}:${IMAGE_TAG}" migrate-down "${PREV_MIGRATION}"; then | |
| echo "::error::migrate-down to ${PREV_MIGRATION} FAILED. Service left STOPPED to avoid running old code against an inconsistent schema. Manual DB recovery required before restarting." | |
| exit 1 | |
| fi | |
| fi | |
| # Restore the EXACT previous image by digest (never a tag that may now resolve to the broken build), | |
| # then scale back to the captured replica count. | |
| if [ -z "${OLD_IMAGE_REF}" ]; then | |
| echo "::error::No previous image captured; cannot auto-restore the image. Manual intervention required." | |
| exit 1 | |
| fi | |
| docker pull "${OLD_IMAGE_REF}" || echo "::warning::could not pull ${OLD_IMAGE_REF}; relying on local copy" | |
| docker service update --force --image "${OLD_IMAGE_REF}" "${SERVICE}" | |
| docker service scale "${SERVICE}=${OLD_REPLICAS:-1}" | |
| # Verify the restored image actually came back healthy. A failed rollback must be distinguishable from a | |
| # failed deploy - never leave the service silently down believing we recovered. | |
| if ! command -v curl >/dev/null 2>&1; then | |
| echo "::warning::curl unavailable on host; cannot verify rollback health. Restored image=${OLD_IMAGE_REF}. Confirm /healthz manually." | |
| exit 0 | |
| fi | |
| echo "Verifying rollback health at ${HEALTH_URL}" | |
| for i in $(seq 1 30); do | |
| if curl -fsS --max-time 5 "${HEALTH_URL}" >/dev/null 2>&1; then | |
| LIVE="$(curl -fsS --max-time 5 "${VERSION_URL}" 2>/dev/null || true)" | |
| echo "Rollback healthy after ${i} check(s); live /version='${LIVE:-<unknown>}'" | |
| exit 0 | |
| fi | |
| echo "Rollback health ${i}/30 not ready; waiting 5s" | |
| sleep 5 | |
| done | |
| echo "::error::Service did NOT become healthy after rollback. Manual intervention required." | |
| exit 1 |