Fixes #1875 - Add Nginx SMTP dummy

docker/batch-test.env

@@ -81,6 +81,8 @@ WEBSERVER_PORT=80
 WEBSERVER_PORT_TLS=443
 WEBSERVER_PORT_IPV6=8080
 WEBSERVER_PORT_IPV6_TLS=4443
+SMTPSERVER_PORT=25
+SMTPSERVER_PORT_IPV6=2525
 UNBOUND_PORT_TCP=53/tcp
 UNBOUND_PORT_UDP=53/udp
 # use fake port numbers here so we don't end up with duplicates in the compose file which causes an error

docker/build.env

@@ -8,6 +8,8 @@ COMPOSE_PROFILES=monitoring,routinator,run-tests
 # don't expose HTTP(S) and DNS ports to the outside, this also causes issues due to being privileged ports
 WEBSERVER_PORT=80
 WEBSERVER_PORT_TLS=443
+SMTPSERVER_PORT=25
+SMTPSERVER_PORT_IPV6=2525
 UNBOUND_PORT_TCP=53/tcp
 UNBOUND_PORT_UDP=53/udp
 # use fake port numbers here so we don't end up with duplicates in the compose file which causes an error

docker/compose.yaml

@@ -27,6 +27,8 @@ services:
       - $WEBSERVER_PORT_IPV6/tcp
       - $WEBSERVER_PORT_IPV6_TLS/tcp
       - $WEBSERVER_PORT_IPV6_TLS/udp
+      - $SMTPSERVER_PORT
+      - $SMTPSERVER_PORT_IPV6
 
     environment:
       - INTERNETNL_DOMAINNAME
@@ -46,6 +48,8 @@ services:
       - NGINX_PROXY_CACHE
       - INTERNETNL_BRANDING
       - LANGUAGES
+      - SMTP_AUTH_HTTP_ENDPOINT
+      - SMTP_EHLO_DOMAIN
 
     # webserver does not depend on any of the other services directly. So it can
     # be started and kept running independently from the other services to

docker/defaults.env

@@ -172,6 +172,8 @@ WEBSERVER_PORT=0.0.0.0:80:80
 WEBSERVER_PORT_TLS=0.0.0.0:443:443
 WEBSERVER_PORT_IPV6=::0:80:80
 WEBSERVER_PORT_IPV6_TLS=::0:443:443
+SMTPSERVER_PORT=0.0.0.0:25:25
+SMTPSERVER_PORT_IPV6=::0:25:25
 UNBOUND_PORT_TCP=0.0.0.0:53:53/tcp
 UNBOUND_PORT_UDP=0.0.0.0:53:53/udp
 UNBOUND_PORT_IPV6_TCP=::0:53:53/tcp
@@ -180,6 +182,9 @@ UNBOUND_PORT_IPV6_UDP=::0:53:53/udp
 # don't export rabbitmq GUI
 RABBITMQ_GUI=127.0.0.1:15672:15672
 
+# nginx internal SMTP auth_http endpoint
+SMTP_AUTH_HTTP_ENDPOINT=127.0.0.1:9001
+
 # configure url to use for public suffix list, empty for default
 PUBLIC_SUFFIX_LIST_URL=
 

docker/develop.env

@@ -49,6 +49,8 @@ WEBSERVER_PORT=80
 WEBSERVER_PORT_TLS=443
 WEBSERVER_PORT_IPV6=8080
 WEBSERVER_PORT_IPV6_TLS=4443
+SMTPSERVER_PORT=25
+SMTPSERVER_PORT_IPV6=2525
 UNBOUND_PORT_TCP=53/tcp
 UNBOUND_PORT_UDP=53/udp
 # use fake port numbers here so we don't end up with duplicates in the compose file which causes an error

docker/host-multi-dist.env

@@ -35,6 +35,9 @@ WEBSERVER_PORT_TLS=$IPV4_IP_PUBLIC:443:443
 WEBSERVER_PORT_IPV6=$IPV6_IP_PUBLIC:80:80
 WEBSERVER_PORT_IPV6_TLS=[$IPV6_IP_PUBLIC]:443:443
 
+SMTPSERVER_PORT=$IPV4_IP_PUBLIC:25:25
+SMTPSERVER_PORT_IPV6=[$IPV6_IP_PUBLIC]:25:25
+
 IPV4_SUBNET_PUBLIC=$IPV4_SUBNET_PUBLIC
 IPV4_SUBNET_INTERNAL=$IPV4_SUBNET_INTERNAL
 

docker/test.env

@@ -80,6 +80,8 @@ WEBSERVER_PORT=80
 WEBSERVER_PORT_TLS=443
 WEBSERVER_PORT_IPV6=8080
 WEBSERVER_PORT_IPV6_TLS=4443
+SMTPSERVER_PORT=25
+SMTPSERVER_PORT_IPV6=2525
 UNBOUND_PORT_TCP=53/tcp
 UNBOUND_PORT_UDP=53/udp
 # use fake port numbers here so we don't end up with duplicates in the compose file which causes an error

docker/webserver.Dockerfile

@@ -21,6 +21,7 @@ RUN mkdir -p /etc/nginx/htpasswd/
 RUN touch /etc/nginx/htpasswd/monitoring.htpasswd
 
 COPY docker/webserver/10-variables.envsh /docker-entrypoint.d/
+COPY docker/webserver/20-envsubst-on-templates.sh /docker-entrypoint.d/
 COPY docker/webserver/tls_init.sh /docker-entrypoint.d/
 COPY docker/webserver/authentication.sh /docker-entrypoint.d/
 COPY docker/webserver/generate_quic_host_key.sh /docker-entrypoint.d/

docker/webserver/20-envsubst-on-templates.sh

@@ -0,0 +1,88 @@
+#!/bin/sh
+
+set -e
+
+ME=$(basename "$0")
+
+entrypoint_log() {
+    if [ -z "${NGINX_ENTRYPOINT_QUIET_LOGS:-}" ]; then
+        echo "$@"
+    fi
+}
+
+ensure_output_writable() {
+  local test_dir=$1
+  if [ ! -w "$test_dir" ]; then
+    entrypoint_log "$ME: ERROR: $template_dir exists, but $test_dir is not writable"
+    exit 0
+  fi
+}
+
+add_extra_block() {
+  local extra=$1
+  local extra_output_dir=$2
+  local conffile="/etc/nginx/nginx.conf"
+
+  if grep -q -E "\s*$extra\s*\{" "$conffile"; then
+    entrypoint_log "$ME: $conffile contains a $extra block; include $extra_output_dir/*.conf to enable $extra templates"
+  else
+    # check if the file can be modified, e.g. not on a r/o filesystem
+    touch "$conffile" 2>/dev/null || { entrypoint_log "$ME: info: can not modify $conffile (read-only file system?)"; exit 0; }
+    entrypoint_log "$ME: Appending $extra block to $conffile to include $extra_output_dir/*.conf"
+    cat << END >> "$conffile"
+# added by "$ME" on "$(date)"
+$extra {
+  include $extra_output_dir/*.conf;
+}
+END
+  fi
+}
+
+write_template_conf() {
+  local select_suffix=$1
+  local conf_output_dir=$2
+  local template relative_path output_path subdir
+  find "$template_dir" -follow -type f -name "*$select_suffix" -print | while read -r template; do
+    relative_path="${template#"$template_dir/"}"
+    output_path="$conf_output_dir/${relative_path%"$select_suffix"}"
+    subdir=$(dirname "$relative_path")
+    # create a subdirectory where the template file exists
+    mkdir -p "$conf_output_dir/$subdir"
+    entrypoint_log "$ME: Running envsubst on $template to $output_path"
+    envsubst "$defined_envs" < "$template" > "$output_path"
+  done
+}
+
+auto_envsubst() {
+  local template_dir="${NGINX_ENVSUBST_TEMPLATE_DIR:-/etc/nginx/templates}"
+  local suffix="${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.template}"
+  local output_dir="${NGINX_ENVSUBST_OUTPUT_DIR:-/etc/nginx/conf.d}"
+  local mail_suffix="${NGINX_ENVSUBST_MAIL_TEMPLATE_SUFFIX:-.mail-template}"
+  local mail_output_dir="${NGINX_ENVSUBST_MAIL_OUTPUT_DIR:-/etc/nginx/mail-conf.d}"
+  local stream_suffix="${NGINX_ENVSUBST_STREAM_TEMPLATE_SUFFIX:-.stream-template}"
+  local stream_output_dir="${NGINX_ENVSUBST_STREAM_OUTPUT_DIR:-/etc/nginx/stream-conf.d}"
+  local filter="${NGINX_ENVSUBST_FILTER:-}"
+
+  local defined_envs=$(printf '${%s} ' $(awk "END { for (name in ENVIRON) { print ( name ~ /${filter}/ ) ? name : \"\" } }" < /dev/null ))
+  [ -d "$template_dir" ] || return 0
+  ensure_output_writable "$output_dir"
+  write_template_conf "$suffix" "$output_dir"
+
+  # Print the first file with the stream suffix, this will be false if there are none
+  if test -n "$(find "$template_dir" -name "*$stream_suffix" -print -quit)"; then
+    mkdir -p "$stream_output_dir"
+    ensure_output_writable "$stream_output_dir"
+    add_extra_block "stream" "$stream_output_dir"
+    write_template_conf "$stream_suffix" "$stream_output_dir"
+  fi
+  if test -n "$(find "$template_dir" -name "*$mail_suffix" -print -quit)"; then
+    mkdir -p "$mail_output_dir"
+    ensure_output_writable "$mail_output_dir"
+    add_extra_block "mail" "$mail_output_dir"
+    write_template_conf "$mail_suffix" "$mail_output_dir"
+  fi
+}
+
+auto_envsubst
+
+exit 0

docker/webserver/nginx_templates/smtp.conf.mail-template

@@ -0,0 +1,13 @@
+server_name ${SMTP_EHLO_DOMAIN};
+auth_http http://${SMTP_AUTH_HTTP_ENDPOINT}/;
+
+starttls only;
+
+include conf.d/tls.conf;
+
+server {
+    listen        25;
+    listen   [::]:25;
+    protocol smtp;
+    smtp_capabilities "SIZE 1099511627776" ENHANCEDSTATUSCODES 8BITMIME DSN SMTPUTF8 REQUIRETLS;
+}

docker/webserver/nginx_templates/smtp_auth_http.conf.template

@@ -0,0 +1,11 @@
+# for mail auth_http
+server {
+    listen ${SMTP_AUTH_HTTP_ENDPOINT};
+    location / {
+        default_type text/plain;
+        add_header Auth-Status "Login not supported since this is a dummy nginx smtp handler";
+        add_header Auth-Error-Code "550 5.3.5";
+        add_header Auth-Wait 1;
+        return 200;
+    }
+}

documentation/Docker-multi-deployment.md

@@ -37,6 +37,8 @@ Add the following lines to `docker/host.env` and change the IP's to the public I
     WEBSERVER_PORT_TLS=192.0.2.2:443:443
     WEBSERVER_PORT_IPV6=[2001:db8:1::2]:80:80/tcp
     WEBSERVER_PORT_IPV6_TLS=[2001:db8:1::2]:443:443/tcp
+    SMTPSERVER_PORT=192.0.2.2:25:25
+    SMTPSERVER_PORT_IPV6=[2001:db8:1::2]:25:25
 
 ## Adding a new instance
 
@@ -52,7 +54,7 @@ Modify the `docker/host.env` file with the following steps:
 - Update `ALLOWED_HOSTS` and `CSP_DEFAULT_SRC` values to the new domain name (eg: `dev2.example.com`)
 - Change `IPV4_IP_PUBLIC`, `IPV6_IP_PUBLIC`, `IPV6_TEST_ADDR` to the public IPv4/IPv6 addresses specific for this instance
 - Update `UNBOUND_PORT_TCP`, `UNBOUND_PORT_UDP`, `UNBOUND_PORT_IPV6_TCP` and `UNBOUND_PORT_IPV6_UDP` to the public IPv4/IPv6 addresses for this instance
-- Add `WEBSERVER_PORT`, `WEBSERVER_PORT_TLS`, `WEBSERVER_PORT_IPV6`, `WEBSERVER_PORT_IPV6_TLS` with the public IPv4/IPv6 addresses for this instance and the respective ports
+- Add `WEBSERVER_PORT`, `WEBSERVER_PORT_TLS`, `WEBSERVER_PORT_IPV6`, `WEBSERVER_PORT_IPV6_TLS`, `SMTPSERVER_PORT`, `SMTPSERVER_PORT_IPV6` with the public IPv4/IPv6 addresses for this instance and the respective ports
 - Add `IPV4_SUBNET_PUBLIC`, `IPV4_SUBNET_INTERNAL`, `IPV6_SUBNET_PUBLIC` and `IPV6_GATEWAY_PUBLIC` with unique subnet/address from private address space, this should not conflict with the existing instances. Suggested is to iterate over subnets for the existing instance (`172.16.42.0/24`, `192.168.42.0/24`, `fd00:42:1::/48`, `fd00:42:1::1`) so the first ones would become: `172.16.43.0/24`, `192.168.43.0/24`, `fd00:43:1::/48` and `fd00:43:1::1`.
 - Add a `ROUTINATOR_URL` with a URL to the first instance routinator proxy endpoint, so the extra instances don't have to run a resource heavy extra routinator, eg: `https://example.com/routinator/api/v1/validity`. This also requires removing the `routinator` entry from `COMPOSE_PROFILES` on the extra instance.
 - Add `INTERNETNL_INSTALL_BASE` with the path to the new instance directory, eg: `/opt/Internet.nl-dev2`

integration_tests/common/test_basic.py

@@ -5,6 +5,7 @@
 from playwright.sync_api import expect
 import socket
 import os
+import time
 
 FOOTER_TEXT_EN = "Internet.nl is an initiative of the Internet community and the Dutch"
 FOOTER_TEXT_NL = "Internet.nl is een initiatief van de internetgemeenschap en de Nederlandse"
@@ -220,3 +221,18 @@ def test_cron_postgres_backups(trigger_cron, docker_compose_exec):
 
     assert docker_compose_exec("cron", "ls /var/lib/postgresql/backups/internetnl_db1.daily.sql.gz")
     assert docker_compose_exec("cron", "ls /var/lib/postgresql/backups/internetnl_db1.weekly.sql.gz")
+
+
+def test_mail_server(app_domain: str):
+    """Test if dummy SMTP server is running, see: https://github.com/internetstandards/Internet.nl/issues/1875."""
+
+    # connect to SMTP socket and test if HELO message is correct
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.connect((app_domain, 25))
+        s.sendall(b"HELO example.com\r\n")
+        time.sleep(1)  # second response line is not sent directly
+        response = s.recv(1024)
+        assert response.decode() == f"220 {app_domain} ESMTP ready\r\n250 {app_domain}\r\n"
+        s.sendall(b"AUTH PLAIN dXNlcgB1c2VyAHB3ZA==\r\n")
+        response = s.recv(1024)
+        assert response == b"500 5.5.1 Invalid command\r\n"