From e9124d499989325639b379daf1f062416c876419 Mon Sep 17 00:00:00 2001 From: "Benjamin W. Broersma" Date: Fri, 3 Oct 2025 22:49:06 +0200 Subject: [PATCH 1/2] Fixes #1875 - Add Nginx SMTP dummy --- docker/batch-test.env | 2 + docker/build.env | 2 + docker/compose.yaml | 4 + docker/defaults.env | 5 ++ docker/develop.env | 2 + docker/host-multi-dist.env | 3 + docker/test.env | 2 + docker/webserver.Dockerfile | 1 + docker/webserver/20-envsubst-on-templates.sh | 88 +++++++++++++++++++ .../nginx_templates/smtp.conf.mail-template | 13 +++ .../smtp_auth_http.conf.template | 11 +++ documentation/Docker-multi-deployment.md | 4 +- 12 files changed, 136 insertions(+), 1 deletion(-) create mode 100755 docker/webserver/20-envsubst-on-templates.sh create mode 100644 docker/webserver/nginx_templates/smtp.conf.mail-template create mode 100644 docker/webserver/nginx_templates/smtp_auth_http.conf.template diff --git a/docker/batch-test.env b/docker/batch-test.env index ba3cb8604..d33261024 100644 --- a/docker/batch-test.env +++ b/docker/batch-test.env @@ -81,6 +81,8 @@ WEBSERVER_PORT=80 WEBSERVER_PORT_TLS=443 WEBSERVER_PORT_IPV6=8080 WEBSERVER_PORT_IPV6_TLS=4443 +SMTPSERVER_PORT=25 +SMTPSERVER_PORT_IPV6=2525 UNBOUND_PORT_TCP=53/tcp UNBOUND_PORT_UDP=53/udp # use fake port numbers here so we don't end up with duplicates in the compose file which causes an error diff --git a/docker/build.env b/docker/build.env index 24b535dc1..e68c294b4 100644 --- a/docker/build.env +++ b/docker/build.env @@ -8,6 +8,8 @@ COMPOSE_PROFILES=monitoring,routinator,run-tests # don't expose HTTP(S) and DNS ports to the outside, this also causes issues due to being privileged ports WEBSERVER_PORT=80 WEBSERVER_PORT_TLS=443 +SMTPSERVER_PORT=25 +SMTPSERVER_PORT_IPV6=2525 UNBOUND_PORT_TCP=53/tcp UNBOUND_PORT_UDP=53/udp # use fake port numbers here so we don't end up with duplicates in the compose file which causes an error diff --git a/docker/compose.yaml b/docker/compose.yaml index b0ab03113..40d693af8 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -27,6 +27,8 @@ services: - $WEBSERVER_PORT_IPV6/tcp - $WEBSERVER_PORT_IPV6_TLS/tcp - $WEBSERVER_PORT_IPV6_TLS/udp + - $SMTPSERVER_PORT + - $SMTPSERVER_PORT_IPV6 environment: - INTERNETNL_DOMAINNAME @@ -46,6 +48,8 @@ services: - NGINX_PROXY_CACHE - INTERNETNL_BRANDING - LANGUAGES + - SMTP_AUTH_HTTP_ENDPOINT + - SMTP_EHLO_DOMAIN # webserver does not depend on any of the other services directly. So it can # be started and kept running independently from the other services to diff --git a/docker/defaults.env b/docker/defaults.env index 31b0aad7d..ac1f2ecf9 100644 --- a/docker/defaults.env +++ b/docker/defaults.env @@ -172,6 +172,8 @@ WEBSERVER_PORT=0.0.0.0:80:80 WEBSERVER_PORT_TLS=0.0.0.0:443:443 WEBSERVER_PORT_IPV6=::0:80:80 WEBSERVER_PORT_IPV6_TLS=::0:443:443 +SMTPSERVER_PORT=0.0.0.0:25:25 +SMTPSERVER_PORT_IPV6=::0:25:25 UNBOUND_PORT_TCP=0.0.0.0:53:53/tcp UNBOUND_PORT_UDP=0.0.0.0:53:53/udp UNBOUND_PORT_IPV6_TCP=::0:53:53/tcp @@ -180,6 +182,9 @@ UNBOUND_PORT_IPV6_UDP=::0:53:53/udp # don't export rabbitmq GUI RABBITMQ_GUI=127.0.0.1:15672:15672 +# nginx internal SMTP auth_http endpoint +SMTP_AUTH_HTTP_ENDPOINT=127.0.0.1:9001 + # configure url to use for public suffix list, empty for default PUBLIC_SUFFIX_LIST_URL= diff --git a/docker/develop.env b/docker/develop.env index 1dfe84f3f..57a18b54b 100644 --- a/docker/develop.env +++ b/docker/develop.env @@ -49,6 +49,8 @@ WEBSERVER_PORT=80 WEBSERVER_PORT_TLS=443 WEBSERVER_PORT_IPV6=8080 WEBSERVER_PORT_IPV6_TLS=4443 +SMTPSERVER_PORT=25 +SMTPSERVER_PORT_IPV6=2525 UNBOUND_PORT_TCP=53/tcp UNBOUND_PORT_UDP=53/udp # use fake port numbers here so we don't end up with duplicates in the compose file which causes an error diff --git a/docker/host-multi-dist.env b/docker/host-multi-dist.env index d57df583f..0f4a3b1c6 100644 --- a/docker/host-multi-dist.env +++ b/docker/host-multi-dist.env @@ -35,6 +35,9 @@ WEBSERVER_PORT_TLS=$IPV4_IP_PUBLIC:443:443 WEBSERVER_PORT_IPV6=$IPV6_IP_PUBLIC:80:80 WEBSERVER_PORT_IPV6_TLS=[$IPV6_IP_PUBLIC]:443:443 +SMTPSERVER_PORT=$IPV4_IP_PUBLIC:25:25 +SMTPSERVER_PORT_IPV6=[$IPV6_IP_PUBLIC]:25:25 + IPV4_SUBNET_PUBLIC=$IPV4_SUBNET_PUBLIC IPV4_SUBNET_INTERNAL=$IPV4_SUBNET_INTERNAL diff --git a/docker/test.env b/docker/test.env index 13c2cbf11..95d16bf86 100644 --- a/docker/test.env +++ b/docker/test.env @@ -80,6 +80,8 @@ WEBSERVER_PORT=80 WEBSERVER_PORT_TLS=443 WEBSERVER_PORT_IPV6=8080 WEBSERVER_PORT_IPV6_TLS=4443 +SMTPSERVER_PORT=25 +SMTPSERVER_PORT_IPV6=2525 UNBOUND_PORT_TCP=53/tcp UNBOUND_PORT_UDP=53/udp # use fake port numbers here so we don't end up with duplicates in the compose file which causes an error diff --git a/docker/webserver.Dockerfile b/docker/webserver.Dockerfile index 63c4edeea..0b822080e 100644 --- a/docker/webserver.Dockerfile +++ b/docker/webserver.Dockerfile @@ -21,6 +21,7 @@ RUN mkdir -p /etc/nginx/htpasswd/ RUN touch /etc/nginx/htpasswd/monitoring.htpasswd COPY docker/webserver/10-variables.envsh /docker-entrypoint.d/ +COPY docker/webserver/20-envsubst-on-templates.sh /docker-entrypoint.d/ COPY docker/webserver/tls_init.sh /docker-entrypoint.d/ COPY docker/webserver/authentication.sh /docker-entrypoint.d/ COPY docker/webserver/generate_quic_host_key.sh /docker-entrypoint.d/ diff --git a/docker/webserver/20-envsubst-on-templates.sh b/docker/webserver/20-envsubst-on-templates.sh new file mode 100755 index 000000000..48c6ca82b --- /dev/null +++ b/docker/webserver/20-envsubst-on-templates.sh @@ -0,0 +1,88 @@ +#!/bin/sh + +set -e + +ME=$(basename "$0") + +entrypoint_log() { + if [ -z "${NGINX_ENTRYPOINT_QUIET_LOGS:-}" ]; then + echo "$@" + fi +} + +ensure_output_writable() { + local test_dir=$1 + if [ ! -w "$test_dir" ]; then + entrypoint_log "$ME: ERROR: $template_dir exists, but $test_dir is not writable" + exit 0 + fi +} + +add_extra_block() { + local extra=$1 + local extra_output_dir=$2 + local conffile="/etc/nginx/nginx.conf" + + if grep -q -E "\s*$extra\s*\{" "$conffile"; then + entrypoint_log "$ME: $conffile contains a $extra block; include $extra_output_dir/*.conf to enable $extra templates" + else + # check if the file can be modified, e.g. not on a r/o filesystem + touch "$conffile" 2>/dev/null || { entrypoint_log "$ME: info: can not modify $conffile (read-only file system?)"; exit 0; } + entrypoint_log "$ME: Appending $extra block to $conffile to include $extra_output_dir/*.conf" + cat << END >> "$conffile" +# added by "$ME" on "$(date)" +$extra { + include $extra_output_dir/*.conf; +} +END + fi +} + +write_template_conf() { + local select_suffix=$1 + local conf_output_dir=$2 + local template relative_path output_path subdir + find "$template_dir" -follow -type f -name "*$select_suffix" -print | while read -r template; do + relative_path="${template#"$template_dir/"}" + output_path="$conf_output_dir/${relative_path%"$select_suffix"}" + subdir=$(dirname "$relative_path") + # create a subdirectory where the template file exists + mkdir -p "$conf_output_dir/$subdir" + entrypoint_log "$ME: Running envsubst on $template to $output_path" + envsubst "$defined_envs" < "$template" > "$output_path" + done +} + +auto_envsubst() { + local template_dir="${NGINX_ENVSUBST_TEMPLATE_DIR:-/etc/nginx/templates}" + local suffix="${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.template}" + local output_dir="${NGINX_ENVSUBST_OUTPUT_DIR:-/etc/nginx/conf.d}" + local mail_suffix="${NGINX_ENVSUBST_MAIL_TEMPLATE_SUFFIX:-.mail-template}" + local mail_output_dir="${NGINX_ENVSUBST_MAIL_OUTPUT_DIR:-/etc/nginx/mail-conf.d}" + local stream_suffix="${NGINX_ENVSUBST_STREAM_TEMPLATE_SUFFIX:-.stream-template}" + local stream_output_dir="${NGINX_ENVSUBST_STREAM_OUTPUT_DIR:-/etc/nginx/stream-conf.d}" + local filter="${NGINX_ENVSUBST_FILTER:-}" + + local defined_envs=$(printf '${%s} ' $(awk "END { for (name in ENVIRON) { print ( name ~ /${filter}/ ) ? name : \"\" } }" < /dev/null )) + [ -d "$template_dir" ] || return 0 + ensure_output_writable "$output_dir" + write_template_conf "$suffix" "$output_dir" + + # Print the first file with the stream suffix, this will be false if there are none + if test -n "$(find "$template_dir" -name "*$stream_suffix" -print -quit)"; then + mkdir -p "$stream_output_dir" + ensure_output_writable "$stream_output_dir" + add_extra_block "stream" "$stream_output_dir" + write_template_conf "$stream_suffix" "$stream_output_dir" + fi + if test -n "$(find "$template_dir" -name "*$mail_suffix" -print -quit)"; then + mkdir -p "$mail_output_dir" + ensure_output_writable "$mail_output_dir" + add_extra_block "mail" "$mail_output_dir" + write_template_conf "$mail_suffix" "$mail_output_dir" + fi +} + +auto_envsubst + +exit 0 diff --git a/docker/webserver/nginx_templates/smtp.conf.mail-template b/docker/webserver/nginx_templates/smtp.conf.mail-template new file mode 100644 index 000000000..d534abcca --- /dev/null +++ b/docker/webserver/nginx_templates/smtp.conf.mail-template @@ -0,0 +1,13 @@ +server_name ${SMTP_EHLO_DOMAIN}; +auth_http http://${SMTP_AUTH_HTTP_ENDPOINT}/; + +starttls only; + +include conf.d/tls.conf; + +server { + listen 25; + listen [::]:25; + protocol smtp; + smtp_capabilities "SIZE 1099511627776" ENHANCEDSTATUSCODES 8BITMIME DSN SMTPUTF8 REQUIRETLS; +} diff --git a/docker/webserver/nginx_templates/smtp_auth_http.conf.template b/docker/webserver/nginx_templates/smtp_auth_http.conf.template new file mode 100644 index 000000000..b39e16800 --- /dev/null +++ b/docker/webserver/nginx_templates/smtp_auth_http.conf.template @@ -0,0 +1,11 @@ +# for mail auth_http +server { + listen ${SMTP_AUTH_HTTP_ENDPOINT}; + location / { + default_type text/plain; + add_header Auth-Status "Login not supported since this is a dummy nginx smtp handler"; + add_header Auth-Error-Code "550 5.3.5"; + add_header Auth-Wait 1; + return 200; + } +} diff --git a/documentation/Docker-multi-deployment.md b/documentation/Docker-multi-deployment.md index b96601b3c..8053acb7f 100644 --- a/documentation/Docker-multi-deployment.md +++ b/documentation/Docker-multi-deployment.md @@ -37,6 +37,8 @@ Add the following lines to `docker/host.env` and change the IP's to the public I WEBSERVER_PORT_TLS=192.0.2.2:443:443 WEBSERVER_PORT_IPV6=[2001:db8:1::2]:80:80/tcp WEBSERVER_PORT_IPV6_TLS=[2001:db8:1::2]:443:443/tcp + SMTPSERVER_PORT=192.0.2.2:25:25 + SMTPSERVER_PORT_IPV6=[2001:db8:1::2]:25:25 ## Adding a new instance @@ -52,7 +54,7 @@ Modify the `docker/host.env` file with the following steps: - Update `ALLOWED_HOSTS` and `CSP_DEFAULT_SRC` values to the new domain name (eg: `dev2.example.com`) - Change `IPV4_IP_PUBLIC`, `IPV6_IP_PUBLIC`, `IPV6_TEST_ADDR` to the public IPv4/IPv6 addresses specific for this instance - Update `UNBOUND_PORT_TCP`, `UNBOUND_PORT_UDP`, `UNBOUND_PORT_IPV6_TCP` and `UNBOUND_PORT_IPV6_UDP` to the public IPv4/IPv6 addresses for this instance -- Add `WEBSERVER_PORT`, `WEBSERVER_PORT_TLS`, `WEBSERVER_PORT_IPV6`, `WEBSERVER_PORT_IPV6_TLS` with the public IPv4/IPv6 addresses for this instance and the respective ports +- Add `WEBSERVER_PORT`, `WEBSERVER_PORT_TLS`, `WEBSERVER_PORT_IPV6`, `WEBSERVER_PORT_IPV6_TLS`, `SMTPSERVER_PORT`, `SMTPSERVER_PORT_IPV6` with the public IPv4/IPv6 addresses for this instance and the respective ports - Add `IPV4_SUBNET_PUBLIC`, `IPV4_SUBNET_INTERNAL`, `IPV6_SUBNET_PUBLIC` and `IPV6_GATEWAY_PUBLIC` with unique subnet/address from private address space, this should not conflict with the existing instances. Suggested is to iterate over subnets for the existing instance (`172.16.42.0/24`, `192.168.42.0/24`, `fd00:42:1::/48`, `fd00:42:1::1`) so the first ones would become: `172.16.43.0/24`, `192.168.43.0/24`, `fd00:43:1::/48` and `fd00:43:1::1`. - Add a `ROUTINATOR_URL` with a URL to the first instance routinator proxy endpoint, so the extra instances don't have to run a resource heavy extra routinator, eg: `https://example.com/routinator/api/v1/validity`. This also requires removing the `routinator` entry from `COMPOSE_PROFILES` on the extra instance. - Add `INTERNETNL_INSTALL_BASE` with the path to the new instance directory, eg: `/opt/Internet.nl-dev2` From 915f17af151d00291612b90965cd9dacf086c684 Mon Sep 17 00:00:00 2001 From: Johan Bloemberg Date: Thu, 9 Oct 2025 11:34:31 +0200 Subject: [PATCH 2/2] Add test for dummy smtp --- integration_tests/common/test_basic.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/integration_tests/common/test_basic.py b/integration_tests/common/test_basic.py index dfa3c7c95..4db098e38 100644 --- a/integration_tests/common/test_basic.py +++ b/integration_tests/common/test_basic.py @@ -5,6 +5,7 @@ from playwright.sync_api import expect import socket import os +import time FOOTER_TEXT_EN = "Internet.nl is an initiative of the Internet community and the Dutch" FOOTER_TEXT_NL = "Internet.nl is een initiatief van de internetgemeenschap en de Nederlandse" @@ -220,3 +221,18 @@ def test_cron_postgres_backups(trigger_cron, docker_compose_exec): assert docker_compose_exec("cron", "ls /var/lib/postgresql/backups/internetnl_db1.daily.sql.gz") assert docker_compose_exec("cron", "ls /var/lib/postgresql/backups/internetnl_db1.weekly.sql.gz") + + +def test_mail_server(app_domain: str): + """Test if dummy SMTP server is running, see: https://github.com/internetstandards/Internet.nl/issues/1875.""" + + # connect to SMTP socket and test if HELO message is correct + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect((app_domain, 25)) + s.sendall(b"HELO example.com\r\n") + time.sleep(1) # second response line is not sent directly + response = s.recv(1024) + assert response.decode() == f"220 {app_domain} ESMTP ready\r\n250 {app_domain}\r\n" + s.sendall(b"AUTH PLAIN dXNlcgB1c2VyAHB3ZA==\r\n") + response = s.recv(1024) + assert response == b"500 5.5.1 Invalid command\r\n"