diff --git a/.changeset/yellow-windows-swim.md b/.changeset/yellow-windows-swim.md new file mode 100644 index 0000000000..6f1f20ec31 --- /dev/null +++ b/.changeset/yellow-windows-swim.md @@ -0,0 +1,6 @@ +--- +--- + +Add a shared image for the DAO API databases that derives `max_connections` and +`effective_cache_size` from the container's cgroup memory limit at boot. +Infra-only — no workspace package changes. diff --git a/infra/dao-api-db/Dockerfile.dao-api-db b/infra/dao-api-db/Dockerfile.dao-api-db new file mode 100644 index 0000000000..2483a0814f --- /dev/null +++ b/infra/dao-api-db/Dockerfile.dao-api-db @@ -0,0 +1,23 @@ +# The fleet spans three Postgres majors (16 x8, 17 x4, 18 x5), so the version +# has to come from the service, not the file. Railway exposes service variables +# as build args, so each DB service sets PG_MAJOR to match the major that +# initialised its volume. +# +# Deliberately no default: an unset PG_MAJOR fails the *build*, which leaves the +# currently-running container serving. A wrong default would instead build fine +# and then crashloop on "database files are incompatible with server" — a louder +# failure at a much worse moment. +ARG PG_MAJOR +FROM ghcr.io/railwayapp-templates/postgres-ssl:${PG_MAJOR} + +# The base image's ENTRYPOINT (wrapper.sh) sets up SSL certs and pgbackrest, so +# we don't replace it — entrypoint.dao-api-db.sh derives a couple of settings from +# the container's memory limit and then execs wrapper.sh with them appended. +COPY entrypoint.dao-api-db.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + +# Declaring ENTRYPOINT in a derived image resets the base image's CMD to empty, +# so it has to be repeated verbatim. This is what `ps` shows in a running +# stock DB container: `wrapper.sh postgres --port=5432`. +CMD ["postgres", "--port=5432"] diff --git a/infra/dao-api-db/dao-api-db.railway.toml b/infra/dao-api-db/dao-api-db.railway.toml new file mode 100644 index 0000000000..78182010bd --- /dev/null +++ b/infra/dao-api-db/dao-api-db.railway.toml @@ -0,0 +1,10 @@ +[build] +builder = "DOCKERFILE" +dockerfilePath = "infra/dao-api-db/Dockerfile.dao-api-db" + +[deploy] +# No healthcheckPath: Postgres speaks the wire protocol, not HTTP, so Railway's +# HTTP probe would never pass. Liveness is covered by postgres-exporter's pg_up, +# which feeds the PostgreSQLUnavailable alert in infra/monitoring/alerts.yml. +restartPolicyType = "ON_FAILURE" +restartPolicyMaxRetries = 10 diff --git a/infra/dao-api-db/entrypoint.dao-api-db.sh b/infra/dao-api-db/entrypoint.dao-api-db.sh new file mode 100644 index 0000000000..26543d767f --- /dev/null +++ b/infra/dao-api-db/entrypoint.dao-api-db.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Derive the memory-dependent Postgres settings from the container's own cgroup +# limit, then hand off to the base image's entrypoint with them appended as -c +# flags. +# +# Why derive instead of hardcode: the DAO databases run on four different memory +# tiers (2 / 4 / 6 / 8 GB) off this one image, and Railway plan changes move a +# service between tiers without touching the repo. Reading the limit at boot +# keeps the settings correct through both. +# +# Why -c flags instead of ALTER SYSTEM: command-line options outrank +# postgresql.auto.conf, so this also neutralises the hand-applied +# `max_connections = 500` currently baked into every DAO volume — without +# writing to the volumes, and revertible by pointing a service back at the +# stock image. +set -euo pipefail + +# Smallest tier in the fleet. Used when the limit is unreadable or unset ("max"), +# because under-provisioning degrades and over-provisioning OOMs. +readonly FALLBACK_BYTES=2000000000 + +# $1 = memory limit in bytes. Echoes " ". +derive() { + local bytes=$1 gb mc ecs_mb + + # Non-numeric covers both "max" (no limit set) and a missing cgroup file. + if [[ ! $bytes =~ ^[0-9]+$ ]] || (( bytes == 0 )); then + bytes=$FALLBACK_BYTES + fi + + # Round to the nearest GB: cgroup reports 1999998976 for a 2 GB tier, and + # truncating that to 1 would put every tier a notch too low. + gb=$(( (bytes + 500000000) / 1000000000 )) + + # ~20 backends per GB, floored so Ponder's two default-30 pools always fit and + # capped so a large tier can't authorise more unreclaimable memory than the + # page cache can spare. 500 backends against the 2 GB tier is ~2.5 GB of + # non-reclaimable memory — the OOM path this exists to close. + mc=$(( gb * 20 )) + (( mc < 75 )) && mc=75 + (( mc > 200 )) && mc=200 + + # Planner hint only — allocates nothing. shared_buffers stays at its 128 MB + # default, so effectively all of this is kernel page cache. + ecs_mb=$(( bytes * 65 / 100 / 1048576 )) + + echo "$mc $ecs_mb" +} + +# Asserts the tiers actually in the fleet plus both clamp boundaries. +# Run locally or in CI with: bash entrypoint.dao-api-db.sh --self-test +self_test() { + local failed=0 + check() { + local got + got=$(derive "$1") + if [[ $got != "$2" ]]; then + echo "FAIL: derive($1) = '$got', expected '$2'" >&2 + failed=1 + fi + } + + check 1999998976 "75 1239" # 2 GB tier (nouns, shutter) — floored + check 3999997952 "80 2479" # 4 GB tier (ens, compound, uniswap@dev) + check 5999996928 "120 3719" # 6 GB tier (aave) + check 8000000000 "160 4959" # 8 GB tier (uniswap@prod) + check 1000000000 "75 619" # below the floor + check 16000000000 "200 9918" # above the cap + check "max" "75 1239" # no limit set -> fallback + check 0 "75 1239" # unreadable -> fallback + + (( failed )) && { echo "self-test FAILED" >&2; return 1; } + echo "self-test OK" +} + +if [[ ${1:-} == "--self-test" ]]; then + self_test + exit $? +fi + +limit_bytes=$(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo max) +read -r max_connections effective_cache_size_mb <<<"$(derive "$limit_bytes")" + +# Escape hatches, in case one database needs to deviate without a redeploy of +# the image. +max_connections=${PG_MAX_CONNECTIONS:-$max_connections} +effective_cache_size=${PG_EFFECTIVE_CACHE_SIZE:-${effective_cache_size_mb}MB} + +echo "[anticapture] memory.max=${limit_bytes} -> max_connections=${max_connections} effective_cache_size=${effective_cache_size}" + +exec /usr/local/bin/wrapper.sh "$@" \ + -c max_connections="$max_connections" \ + -c effective_cache_size="$effective_cache_size"