Fix SslStream/QUIC server exit hang; re-enable handshake scenarios (Wave A) — closes #2140 partially

build/containers-scenarios.yml

@@ -110,12 +110,22 @@ parameters:
     arguments: --scenario fortunes_vertx --property scenario=FortunesVertx
     condition: Math.round(Date.now() / 43200000) % 10 == 9 # once every 10 half-days
 
-  - displayName: Json NTex
-    arguments: --scenario json_ntex --property scenario=JsonNTex
-    condition: Math.round(Date.now() / 43200000) % 10 == 8 # once every 10 half-days
-  - displayName: Fortunes NTex
-    arguments: --scenario fortunes_ntex --property scenario=FortunesNTex
-    condition: Math.round(Date.now() / 43200000) % 10 == 9 # once every 10 half-days
+  # Json NTex and Fortunes NTex are disabled: their docker builds
+  # (ntex-plt.dockerfile and ntex-db.dockerfile in TechEmpower/
+  # FrameworkBenchmarks) all fail because the ntex workspace has
+  # `tokio-postgres` as a top-level dep on the fafhrd91/postgres
+  # ntex-3 fork (commit fbc7a17), which is no longer compatible
+  # with newer ntex-bytes resolved from TechEmpower's caret-versioned
+  # `ntex-bytes = "1.5"`. TechEmpower/FrameworkBenchmarks is archived
+  # (https://github.com/TechEmpower/FrameworkBenchmarks), so no fix
+  # is coming upstream. See build/frameworks-scenarios.yml and
+  # build/frameworks-database-scenarios.yml for the matching disables.
+  # - displayName: Json NTex
+  #   arguments: --scenario json_ntex --property scenario=JsonNTex
+  #   condition: Math.round(Date.now() / 43200000) % 10 == 8 # once every 10 half-days
+  # - displayName: Fortunes NTex
+  #   arguments: --scenario fortunes_ntex --property scenario=FortunesNTex
+  #   condition: Math.round(Date.now() / 43200000) % 10 == 9 # once every 10 half-days
 
 steps:
 - ${{ each scenario in parameters.scenarios }}:

build/frameworks-database-scenarios.yml

@@ -52,9 +52,22 @@ parameters:
     arguments: --scenario json_gin --load.variables.connections 512 --property scenario=JsonGin
 
   #  Ntex (Rust)
-
-  - displayName: Ntex Fortunes
-    arguments: --scenario fortunes_ntex --load.variables.connections 512 --property scenario=FortunesNtex
+  #
+  # Ntex Fortunes is disabled: the docker build for ntex-db.dockerfile
+  # is broken because the TechEmpower ntex workspace has `tokio-postgres`
+  # as a top-level dependency pointing at the fafhrd91/postgres `ntex-3`
+  # fork (frozen at commit fbc7a17). That fork no longer compiles against
+  # the latest ntex-bytes (1.6+) that cargo resolves from TechEmpower's
+  # caret-versioned `ntex-bytes = "1.5"`, so every binary in the
+  # workspace - including ntex-db - fails with E0308 mismatched-types
+  # errors on `BytesMut` vs `BytePages`.
+  # TechEmpower/FrameworkBenchmarks is now archived
+  # (https://github.com/TechEmpower/FrameworkBenchmarks), so a real fix
+  # upstream is unlikely and not worth chasing. See
+  # build/frameworks-scenarios.yml for the matching Ntex Plaintext and
+  # Ntex Json disables.
+  # - displayName: Ntex Fortunes
+  #   arguments: --scenario fortunes_ntex --load.variables.connections 512 --property scenario=FortunesNtex
 
   #  Vertx (Java)
 

build/frameworks-scenarios.yml

@@ -41,11 +41,24 @@ parameters:
   # Scenarios are in frameworks-database-scenarios.yml since they require the db server
 
 #  Ntex (Rust)
-
-  - displayName: Ntex Plaintext
-    arguments: --scenario plaintext_ntex --load.variables.connections 1024 --property scenario=PlaintextNtex
-  - displayName: Ntex Json
-    arguments: --scenario json_ntex --load.variables.connections 512 --property scenario=JsonNtex
+#
+# Ntex Plaintext and Ntex Json are disabled: the docker build for
+# ntex-plt.dockerfile is broken because the TechEmpower ntex
+# workspace has `tokio-postgres` as a top-level dependency pointing
+# at the fafhrd91/postgres `ntex-3` fork (frozen at commit fbc7a17).
+# That fork no longer compiles against the latest ntex-bytes (1.6+)
+# that cargo resolves from TechEmpower's caret-versioned
+# `ntex-bytes = "1.5"`, so every binary in the workspace - including
+# ntex-plt - fails with E0308 mismatched-types errors on `BytesMut`
+# vs `BytePages`. TechEmpower/FrameworkBenchmarks is now archived
+# (https://github.com/TechEmpower/FrameworkBenchmarks), so a real
+# fix upstream is unlikely. See build/frameworks-database-scenarios.yml
+# for the matching Ntex Fortunes disable.
+
+  # - displayName: Ntex Plaintext
+  #   arguments: --scenario plaintext_ntex --load.variables.connections 1024 --property scenario=PlaintextNtex
+  # - displayName: Ntex Json
+  #   arguments: --scenario json_ntex --load.variables.connections 512 --property scenario=JsonNtex
 
 #  Vertx (Java)
 

build/sslstream-scenarios.yml

@@ -124,7 +124,7 @@ steps:
                           useDataContractSerializer: "false"
                           messageBody: |
                             {
-                              "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 ) && false",
+                              "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 )",
                               "name": "crank",
                               "args": [ "${{ parameters.tfm }} --client.options.collectCounters true --command-line-property --table SystemNet_TlsBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --description \"${{ t.displayName }} ${{ version.displayName }} ${{ s.displayName }} ${{ c.displayName }} ${{ res.displayName }} $(System.JobDisplayName)\" ${{ parameters.arguments }} --no-metadata --no-measurements ${{ t.arguments }} ${{ s.arguments }} ${{ version.arguments }} ${{ res.arguments }} ${{ c.arguments }}" ]
                             }
@@ -145,7 +145,7 @@ steps:
                   useDataContractSerializer: "false"
                   messageBody: |
                     {
-                      "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 ) && false",
+                      "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 )",
                       "name": "crank",
                       "args": [ "${{ parameters.tfm }} --client.options.collectCounters true --command-line-property --table SystemNet_TlsBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --description \"${{ t.displayName }} ${{ s.displayName }} ${{ c.displayName }} $(System.JobDisplayName)\" ${{ parameters.arguments }} --no-metadata --no-measurements ${{ t.arguments }} ${{ s.arguments }} ${{ c.arguments }}" ]
                     }
@@ -169,7 +169,7 @@ steps:
                               useDataContractSerializer: "false"
                               messageBody: |
                                 {
-                                  "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 ) && false",
+                                  "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 )",
                                   "name": "crank",
                                   "args": [ "${{ parameters.tfm }} --client.options.collectCounters true --command-line-property --table SystemNet_TlsBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --description \"${{ t.displayName }} ${{ version.displayName }} ${{ s.displayName }} ${{ sendbuf.displayName }} ${{ recvbuf.displayName }} ${{ c.displayName }} $(System.JobDisplayName)\" ${{ parameters.arguments }} --no-metadata --no-measurements ${{ t.arguments }} ${{ s.arguments }} ${{ version.arguments }} ${{ sendbuf.arguments }} ${{ recvbuf.arguments }} ${{ c.arguments }}" ]
                                 }
@@ -192,7 +192,7 @@ steps:
                           useDataContractSerializer: "false"
                           messageBody: |
                             {
-                              "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 ) && false",
+                              "condition": "(${{ parameters.condition }}) && (Math.round(Date.now() / 43200000) % 2 == 0 )",
                               "name": "crank",
                               "args": [ "${{ parameters.tfm }} --client.options.collectCounters true --command-line-property --table SystemNet_TlsBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart --session $(session) --description \"${{ t.displayName }} ${{ s.displayName }} ${{ sendbuf.displayName }} ${{ recvbuf.displayName }} ${{ c.displayName }} $(System.JobDisplayName)\" ${{ parameters.arguments }} --no-metadata --no-measurements ${{ t.arguments }} ${{ s.arguments }} ${{ sendbuf.arguments }} ${{ recvbuf.arguments }} ${{ c.arguments }}" ]
                             }

src/System/Net/Benchmarks/Common/Server/BenchmarkServer.cs

@@ -1,12 +1,20 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using System.Collections.Concurrent;
+
 namespace System.Net.Benchmarks;
 
 internal abstract class BenchmarkServer<TListener, TAcceptResult, TOptions> : BenchmarkApp<TOptions>
     where TListener : IListener<TAcceptResult>
     where TOptions : IBenchmarkServerOptions, new()
 {
+    // Maximum time to wait for in-flight accepted-connection tasks to drain after
+    // cancellation before letting the listener dispose. Keep this comfortably under
+    // BenchmarkApp.s_shutdownDeadline so the outer deadline still has room to fire
+    // if disposal itself stalls.
+    private static readonly TimeSpan s_drainTimeout = TimeSpan.FromSeconds(10);
+
     private static int s_errors;
 
     protected abstract Task<TListener> ListenAsync(TOptions options, CancellationToken ct);
@@ -19,17 +27,29 @@ protected override async Task RunBenchmarkAsync(TOptions options, CancellationTo
         await using var listener = await ListenAsync(options, ct);
         Log(GetReadyStateText(listener));
 
+        // Track per-connection tasks so we can drain them on cancellation before
+        // the listener disposes. Without this, `await using var listener` can race
+        // against in-flight TLS/QUIC connections — SslStream.DisposeAsync attempts
+        // to send a close_notify alert, and if the underlying socket is being torn
+        // down concurrently, the dispose can stall long enough to strand the port
+        // for the next benchmark run.
+        var inflight = new ConcurrentDictionary<Task, byte>();
         try
         {
             while (!ct.IsCancellationRequested)
             {
                 var accepted = await listener.AcceptAsync(ct).ConfigureAwait(false);
-                _ = Task.Run(() => ProcessAcceptedNoThrowAsync(accepted, options, ct), ct);
+                var task = Task.Run(() => ProcessAcceptedNoThrowAsync(accepted, options, ct), ct);
+                inflight[task] = 0;
+                _ = task.ContinueWith(static (t, state) => ((ConcurrentDictionary<Task, byte>)state!).TryRemove(t, out _),
+                    inflight, TaskScheduler.Default);
             }
         }
         catch (OperationCanceledException e) when (e.CancellationToken == ct) { }
         finally
         {
+            await DrainAsync(inflight).ConfigureAwait(false);
+
             if (s_errors > 0)
             {
                 LogMetric(MetricName.Errors, "Errors", s_errors, "n0");
@@ -38,6 +58,21 @@ protected override async Task RunBenchmarkAsync(TOptions options, CancellationTo
         Log("Exiting...");
     }
 
+    private static async Task DrainAsync(ConcurrentDictionary<Task, byte> inflight)
+    {
+        if (inflight.IsEmpty)
+        {
+            return;
+        }
+
+        var pending = Task.WhenAll(inflight.Keys);
+        var completed = await Task.WhenAny(pending, Task.Delay(s_drainTimeout)).ConfigureAwait(false);
+        if (completed != pending)
+        {
+            Log($"Drain timeout ({s_drainTimeout.TotalSeconds:n0}s) reached with {inflight.Count} in-flight connection task(s) still running; continuing shutdown.");
+        }
+    }
+
     private async Task ProcessAcceptedNoThrowAsync(TAcceptResult accepted, TOptions options, CancellationToken ct)
     {
         try

src/System/Net/Benchmarks/Common/Shared/BenchmarkApp.cs

@@ -3,11 +3,17 @@
 
 using System.CommandLine;
 using System.CommandLine.Parsing;
+using System.Runtime.InteropServices;
 
 namespace System.Net.Benchmarks;
 
 public abstract class BenchmarkApp<TOptions> where TOptions : new()
 {
+    // Maximum time to wait for graceful shutdown after cancellation is signaled
+    // before forcing process exit. Crank sends SIGTERM, waits 5s, then SIGINT,
+    // so we keep our deadline well under any subsequent SIGKILL.
+    private static readonly TimeSpan s_shutdownDeadline = TimeSpan.FromSeconds(15);
+
     private static bool s_appStarted;
 
     protected static CancellationTokenSource GlobalCts { get; } = new();
@@ -66,6 +72,26 @@ private async Task<int> RunAsyncInternal(TOptions options)
             GlobalCts.Cancel();
         };
 
+        // Crank's Linux agent stops jobs with SIGTERM first (5 s grace) then SIGINT.
+        // Without an explicit SIGTERM handler, the .NET runtime's default behavior
+        // terminates the process after a short ProcessExit window and any in-flight
+        // TLS/QUIC connections never observe cancellation. That can leave sockets
+        // in a half-broken state, stranding the listen port for the next benchmark.
+        // Register SIGTERM/SIGQUIT here so GlobalCts.Cancel runs the same shutdown
+        // path on Linux as Ctrl+C does on Windows.
+        using var sigTerm = PosixSignalRegistration.Create(PosixSignal.SIGTERM, static ctx =>
+        {
+            Log("SIGTERM received...");
+            ctx.Cancel = true;
+            GlobalCts.Cancel();
+        });
+        using var sigQuit = PosixSignalRegistration.Create(PosixSignal.SIGQUIT, static ctx =>
+        {
+            Log("SIGQUIT received...");
+            ctx.Cancel = true;
+            GlobalCts.Cancel();
+        });
+
         // NOTE:
         // It is better for metrics to be registered with some delay to ensure the metadata is collected.
         // Event listener is started (shortly) after the benchmark app starts, so it might miss the registration event.
@@ -76,9 +102,15 @@ private async Task<int> RunAsyncInternal(TOptions options)
             LogHelper.LogMetric("env/processorcount", Environment.ProcessorCount);
         });
 
+        // Run the benchmark, but enforce a hard shutdown deadline once cancellation
+        // has been requested. If RunBenchmarkAsync hangs in disposal (for example a
+        // stuck SslStream close_notify or QuicConnection drain), Environment.Exit
+        // guarantees we release the port for the next run instead of waiting for
+        // crank's SIGKILL fallback.
         try
         {
-            await RunBenchmarkAsync(options, GlobalCts.Token);
+            var benchmarkTask = RunBenchmarkAsync(options, GlobalCts.Token);
+            await WaitWithShutdownDeadlineAsync(benchmarkTask).ConfigureAwait(false);
         }
         catch (Exception e)
         {
@@ -87,4 +119,39 @@ private async Task<int> RunAsyncInternal(TOptions options)
         }
         return 0; // on success, returning 0 explicitly; otherwise RootCommand (?) will not report unhandled exceptions to crank as failures
     }
+
+    private static async Task WaitWithShutdownDeadlineAsync(Task benchmarkTask)
+    {
+        // Fast path: benchmark finished (or threw) before cancellation was ever requested.
+        await Task.WhenAny(benchmarkTask, WaitForCancellationAsync(GlobalCts.Token)).ConfigureAwait(false);
+
+        if (benchmarkTask.IsCompleted)
+        {
+            await benchmarkTask.ConfigureAwait(false);
+            return;
+        }
+
+        // Cancellation has been signaled. Give the benchmark a bounded time to wind
+        // down gracefully, then force-exit so we don't strand the port between runs.
+        var completed = await Task.WhenAny(benchmarkTask, Task.Delay(s_shutdownDeadline)).ConfigureAwait(false);
+        if (completed == benchmarkTask)
+        {
+            await benchmarkTask.ConfigureAwait(false);
+            return;
+        }
+
+        Log($"Shutdown deadline ({s_shutdownDeadline.TotalSeconds:n0}s) exceeded after cancellation; forcing process exit.");
+        Environment.Exit(0);
+    }
+
+    private static Task WaitForCancellationAsync(CancellationToken ct)
+    {
+        if (ct.IsCancellationRequested)
+        {
+            return Task.CompletedTask;
+        }
+        var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        ct.Register(static state => ((TaskCompletionSource)state!).TrySetResult(), tcs);
+        return tcs.Task;
+    }
 }