|
| 1 | +#!/bin/sh |
| 2 | +# |
| 3 | +# Reload nginx a number of times in a row and look for what a reload leaks. |
| 4 | +# |
| 5 | +# Every reload builds a new cycle: a fresh configuration pool, fresh |
| 6 | +# workers, re-opened listening sockets. The old one has to go away. A |
| 7 | +# module that allocates or opens something per cycle and never gives it |
| 8 | +# back is invisible in normal use -- nothing fails, nothing is logged, the |
| 9 | +# process simply grows by one cycle's worth every time somebody runs |
| 10 | +# "nginx -s reload", for the life of the server. A test suite never sees |
| 11 | +# it, because a suite starts nginx once. |
| 12 | +# |
| 13 | +# usage: ci/reload.sh <nginx binary> |
| 14 | +# |
| 15 | +# Set RELOAD_LOAD_MODULE to the path of the .so to exercise a loadable |
| 16 | +# build, RELOADS to a different number of rounds, RELOAD_PORT to move the |
| 17 | +# listener. |
| 18 | +# |
| 19 | +# ci/reload.conf carries the part that differs per module. It is sourced, |
| 20 | +# with $WORK and $PORT already set, and defines: |
| 21 | +# |
| 22 | +# RELOAD_HTTP directives for the http block, may be empty |
| 23 | +# RELOAD_SERVER directives for the server block |
| 24 | +# RELOAD_EXPECT the body the probe below has to return |
| 25 | +# reload_probe() prints what the server answered |
| 26 | +# |
| 27 | +# WHAT IS ASSERTED, and why each one is shaped the way it is: |
| 28 | +# |
| 29 | +# The module still answers after every reload. Portable, needs nothing |
| 30 | +# from the system, and it is the oracle that catches a module whose |
| 31 | +# per-cycle state is gone or stale after the handover. |
| 32 | +# |
| 33 | +# The worker generation drains back to what the configuration asks for. |
| 34 | +# A worker that outlives its cycle IS the leak, in process form: the |
| 35 | +# server ends up carrying one worker per reload it has ever served. |
| 36 | +# |
| 37 | +# The master's descriptor count does not move. A listening socket or a |
| 38 | +# channel descriptor kept from the previous cycle is deterministic and |
| 39 | +# countable, which makes it the sharp oracle; the worker-side view cannot |
| 40 | +# see it, because a worker only ever knows the cycle it was forked into. |
| 41 | +# Needs /proc, so it is skipped VISIBLY where there is none, never |
| 42 | +# silently. |
| 43 | +# |
| 44 | +# There is deliberately NO check on the master's resident size. Measured |
| 45 | +# here, a healthy series grows the master by about twenty-five pages per |
| 46 | +# reload -- the allocator keeping what it has freed -- while a leaked |
| 47 | +# cycle pool is a handful of pages. Any band wide enough not to flap is |
| 48 | +# already wider than the thing it would have to catch, so such a check |
| 49 | +# could never fail for the right reason. The descriptor count above is |
| 50 | +# the sharp oracle and it needs no band at all. |
| 51 | +# |
| 52 | +# No worker died by signal. A crashed and respawned worker looks exactly |
| 53 | +# like a reloaded one from the outside, so the error log is what tells |
| 54 | +# the two apart. |
| 55 | + |
| 56 | +set -eu |
| 57 | + |
| 58 | +NGINX=${1:?path to the nginx binary missing} |
| 59 | +SRC=$(cd "$(dirname "$0")/.." && pwd) |
| 60 | +WORK=${CI_WORK:-$SRC/ci-work}/reload |
| 61 | +PORT=${RELOAD_PORT:-18200} |
| 62 | +RELOADS=${RELOADS:-8} |
| 63 | + |
| 64 | +# Two workers, so that "the generation drained" is a statement about a set |
| 65 | +# of processes and not about a single one that happens to be replaced. |
| 66 | +WORKERS=2 |
| 67 | + |
| 68 | +command -v curl > /dev/null || { echo "curl(1) is needed" >&2; exit 2; } |
| 69 | +command -v pgrep > /dev/null || { echo "pgrep(1) is needed" >&2; exit 2; } |
| 70 | + |
| 71 | +[ -r "$SRC/ci/reload.conf" ] || { |
| 72 | + echo "ci/reload.sh: ci/reload.conf is missing" >&2 |
| 73 | + exit 2 |
| 74 | +} |
| 75 | + |
| 76 | +rm -rf "$WORK" |
| 77 | +mkdir -p "$WORK/conf" "$WORK/logs" |
| 78 | + |
| 79 | +ngx_user= |
| 80 | +if [ "$(id -u)" = 0 ]; then |
| 81 | + ngx_user="user root $(id -gn);" |
| 82 | +fi |
| 83 | + |
| 84 | +ngx_load= |
| 85 | +if [ -n "${RELOAD_LOAD_MODULE:-}" ]; then |
| 86 | + ngx_load="load_module $RELOAD_LOAD_MODULE;" |
| 87 | +fi |
| 88 | + |
| 89 | +# shellcheck source=reload.conf |
| 90 | +. "$SRC/ci/reload.conf" |
| 91 | + |
| 92 | +cat > "$WORK/conf/nginx.conf" <<EOF |
| 93 | +$ngx_load |
| 94 | +$ngx_user |
| 95 | +worker_processes $WORKERS; |
| 96 | +error_log $WORK/logs/error.log info; |
| 97 | +pid $WORK/logs/nginx.pid; |
| 98 | +events { worker_connections 64; } |
| 99 | +http { |
| 100 | + access_log off; |
| 101 | +$RELOAD_HTTP |
| 102 | +
|
| 103 | + server { |
| 104 | + listen 127.0.0.1:$PORT; |
| 105 | +$RELOAD_SERVER |
| 106 | + } |
| 107 | +} |
| 108 | +EOF |
| 109 | + |
| 110 | +fail=0 |
| 111 | +note() { |
| 112 | + echo " $1" >&2 |
| 113 | + fail=1 |
| 114 | +} |
| 115 | + |
| 116 | +echo "--- the configuration is read ---" |
| 117 | +"$NGINX" -p "$WORK" -c conf/nginx.conf -t |
| 118 | + |
| 119 | +"$NGINX" -p "$WORK" -c conf/nginx.conf |
| 120 | +sleep 1 |
| 121 | + |
| 122 | +MASTER=$(cat "$WORK/logs/nginx.pid") |
| 123 | + |
| 124 | +stop() { |
| 125 | + [ -s "$WORK/logs/nginx.pid" ] && kill "$(cat "$WORK/logs/nginx.pid")" 2> /dev/null |
| 126 | + return 0 |
| 127 | +} |
| 128 | +trap stop EXIT |
| 129 | + |
| 130 | +# The set of WORKER pids, sorted, as one line. Used to tell a finished |
| 131 | +# handover from one still in flight: while the old generation drains, both |
| 132 | +# sets are present and every count taken then is noise. |
| 133 | +# |
| 134 | +# Matched on the process title rather than simply taken as "every child of |
| 135 | +# the master": a module that asks for a cache starts a cache manager and a |
| 136 | +# cache loader, and those are children too. Counting them made this wait |
| 137 | +# for a number of workers that could never be reached. |
| 138 | +workers() { |
| 139 | + pgrep -P "$MASTER" -f 'worker process' 2> /dev/null | sort | tr '\n' ' ' |
| 140 | +} |
| 141 | + |
| 142 | +master_fds() { |
| 143 | + [ -r "/proc/$MASTER/fd" ] || return 1 |
| 144 | + # shellcheck disable=SC2012 # a count of entries, and those names are |
| 145 | + # decimal integers, so they cannot contain a newline |
| 146 | + ls "/proc/$MASTER/fd" 2> /dev/null | wc -l | tr -d ' ' |
| 147 | +} |
| 148 | + |
| 149 | +# Wait until the master has exactly $WORKERS children and none of them is |
| 150 | +# one of the processes that served the previous cycle. |
| 151 | +await_generation() { |
| 152 | + before=$1 |
| 153 | + n=0 |
| 154 | + while [ $n -lt 100 ]; do |
| 155 | + now=$(workers) |
| 156 | + count=$(echo "$now" | wc -w | tr -d ' ') |
| 157 | + if [ "$count" = "$WORKERS" ] && [ "$now" != "$before" ]; then |
| 158 | + overlap=0 |
| 159 | + for p in $now; do |
| 160 | + case " $before " in |
| 161 | + *" $p "*) overlap=1 ;; |
| 162 | + esac |
| 163 | + done |
| 164 | + [ "$overlap" = 0 ] && return 0 |
| 165 | + fi |
| 166 | + n=$((n + 1)) |
| 167 | + sleep 0.1 2> /dev/null || sleep 1 |
| 168 | + done |
| 169 | + return 1 |
| 170 | +} |
| 171 | + |
| 172 | +echo "--- the module answers before the first reload ---" |
| 173 | +got=$(reload_probe || true) |
| 174 | +if [ "$got" = "$RELOAD_EXPECT" ]; then |
| 175 | + echo " $got" |
| 176 | +else |
| 177 | + note "expected \"$RELOAD_EXPECT\", got \"$got\"" |
| 178 | + exit 1 |
| 179 | +fi |
| 180 | + |
| 181 | +FDS_BASE=$(master_fds || true) |
| 182 | +GEN=$(workers) |
| 183 | + |
| 184 | +echo "--- $RELOADS reloads ---" |
| 185 | +absorbed=0 |
| 186 | +answered=0 |
| 187 | +r=1 |
| 188 | +while [ $r -le "$RELOADS" ]; do |
| 189 | + "$NGINX" -p "$WORK" -c conf/nginx.conf -s reload |
| 190 | + if await_generation "$GEN"; then |
| 191 | + absorbed=$((absorbed + 1)) |
| 192 | + else |
| 193 | + note "reload $r: no fresh worker generation within 10 s" |
| 194 | + r=$((r + 1)) |
| 195 | + continue |
| 196 | + fi |
| 197 | + GEN=$(workers) |
| 198 | + |
| 199 | + got=$(reload_probe || true) |
| 200 | + if [ "$got" = "$RELOAD_EXPECT" ]; then |
| 201 | + answered=$((answered + 1)) |
| 202 | + else |
| 203 | + note "reload $r: expected \"$RELOAD_EXPECT\", got \"$got\"" |
| 204 | + fi |
| 205 | + r=$((r + 1)) |
| 206 | +done |
| 207 | +echo " $absorbed of $RELOADS absorbed, $answered answered correctly" |
| 208 | + |
| 209 | +# "absorbed" already means the old generation was gone and a full new one |
| 210 | +# had answered, so a separate drain assertion would be the same claim twice. |
| 211 | +[ "$absorbed" = "$RELOADS" ] || note "not every reload was absorbed" |
| 212 | +[ "$answered" = "$RELOADS" ] || note "the module did not answer after every reload" |
| 213 | + |
| 214 | +echo "--- the master kept its descriptors ---" |
| 215 | +FDS_END=$(master_fds || true) |
| 216 | +if [ -z "$FDS_BASE" ] || [ -z "$FDS_END" ]; then |
| 217 | + echo " skipped, /proc/$MASTER/fd is not readable here" |
| 218 | +elif [ "$FDS_END" = "$FDS_BASE" ]; then |
| 219 | + echo " $FDS_END before and after $RELOADS reloads" |
| 220 | +else |
| 221 | + note "the count moved $FDS_BASE -> $FDS_END over $RELOADS reloads" |
| 222 | +fi |
| 223 | + |
| 224 | +echo "--- no worker died by signal ---" |
| 225 | +if grep -qE 'exited on signal|SIGSEGV|SIGABRT|SIGBUS' "$WORK/logs/error.log" 2> /dev/null; then |
| 226 | + grep -nE 'exited on signal|SIGSEGV|SIGABRT|SIGBUS' "$WORK/logs/error.log" | head -5 >&2 |
| 227 | + note "see the lines above" |
| 228 | +else |
| 229 | + echo " none" |
| 230 | +fi |
| 231 | + |
| 232 | +echo "--- nginx is still running ---" |
| 233 | +if kill -0 "$MASTER" 2> /dev/null; then |
| 234 | + echo " yes" |
| 235 | +else |
| 236 | + note "the master is gone" |
| 237 | +fi |
| 238 | + |
| 239 | +if grep -qE '\[alert\]|\[crit\]|\[emerg\]' "$WORK/logs/error.log" 2> /dev/null; then |
| 240 | + echo "--- the error log complains ---" >&2 |
| 241 | + grep -E '\[alert\]|\[crit\]|\[emerg\]' "$WORK/logs/error.log" | head -5 >&2 |
| 242 | + fail=1 |
| 243 | +fi |
| 244 | + |
| 245 | +[ "$fail" -eq 0 ] |
0 commit comments