Send SERVFAIL response on DNS query failure to prevent client timeouts

When DnsQueryRaw fails (synchronously or asynchronously), the Windows DNS
resolver currently sends no response back to the Linux DNS client. This
forces the client to wait for its full retransmit timeout (typically 5-10
seconds per query) before moving on, causing significant DNS delays.

This change sends a minimal 12-byte DNS SERVFAIL response (RFC 1035,
RCODE=2) back to the Linux DNS client whenever:
- DnsQueryRaw returns a synchronous failure (not DNS_REQUEST_PENDING)
- The async completion callback receives null results
- The async completion returns a non-null result with no raw response

The SERVFAIL response uses the original request's transaction ID so the
client can match it to the pending query and immediately retry or fail.

This also fixes a memory leak on the Linux side: when no response was
sent, UDP request tracking entries in DnsServer::m_udpRequests were never
cleaned up. The SERVFAIL response now triggers the normal response
handling path which erases the tracking entry.

Refs: #4285 #5256 #4737

Author: syed-dawood

src/windows/common/DnsResolver.cpp

@@ -223,6 +223,12 @@ try
     auto [it, _] = m_dnsRequests.emplace(requestId, std::move(context));
     const auto localContext = it->second.get();
 
+    // Store the DNS transaction ID (first 2 bytes of the request) for constructing SERVFAIL responses
+    if (dnsBuffer.size() >= 2)
+    {
+        memcpy(&localContext->m_dnsTransactionId, dnsBuffer.data(), sizeof(localContext->m_dnsTransactionId));
+    }
+
     auto removeContextOnError = wil::scope_exit([&] { WI_VERIFY(m_dnsRequests.erase(requestId) == 1); });
 
     // Fill DNS request structure
@@ -264,6 +270,10 @@ try
             TraceLoggingValue(requestId, "requestId"),
             TraceLoggingValue(result, "result"),
             TraceLoggingValue("DnsQueryRaw", "executionStep"));
+
+        // Send SERVFAIL back to Linux so the DNS client gets an immediate error
+        // instead of waiting for a timeout.
+        SendServfailResponse(localContext->m_dnsTransactionId, localContext->m_dnsClientIdentifier);
         return;
     }
 
@@ -337,6 +347,12 @@ try
             m_dnsChannel.SendDnsMessage(gsl::make_span(dnsResponse), dnsClientIdentifier);
         });
     }
+    else if (!m_stopped)
+    {
+        // The Windows DNS API failed to resolve the request. Send a SERVFAIL response to the Linux DNS client
+        // so it gets an immediate error instead of waiting for a timeout (which can take 5-10 seconds).
+        SendServfailResponse(queryContext->m_dnsTransactionId, queryContext->m_dnsClientIdentifier);
+    }
 
     // Stop tracking this DNS request and delete the request context
     WI_VERIFY(m_dnsRequests.erase(queryContext->m_id) == 1);
@@ -349,6 +365,26 @@ try
 }
 CATCH_LOG()
 
+void DnsResolver::SendServfailResponse(uint16_t transactionId, const LX_GNS_DNS_CLIENT_IDENTIFIER& dnsClientIdentifier)
+{
+    // Build a minimal 12-byte DNS SERVFAIL response per RFC 1035 section 4.1.1.
+    // This allows the Linux DNS client to immediately learn the query failed,
+    // rather than waiting for a retransmit timeout (typically 5-10 seconds).
+    std::vector<gsl::byte> servfail(12, gsl::byte{0});
+    memcpy(servfail.data(), &transactionId, sizeof(transactionId)); // Transaction ID (network byte order, copied as-is)
+    servfail[2] = gsl::byte{0x80};                                  // QR=1 (response), OPCODE=0 (standard query)
+    servfail[3] = gsl::byte{0x02};                                  // RA=0, Z=0, RCODE=2 (Server Failure)
+
+    WSL_LOG(
+        "DnsResolver::SendServfailResponse",
+        TraceLoggingValue(dnsClientIdentifier.Protocol == IPPROTO_UDP ? "UDP" : "TCP", "Protocol"),
+        TraceLoggingValue(dnsClientIdentifier.DnsClientId, "DNS client id"));
+
+    m_dnsResponseQueue.submit([this, servfail = std::move(servfail), dnsClientIdentifier]() mutable {
+        m_dnsChannel.SendDnsMessage(gsl::make_span(servfail), dnsClientIdentifier);
+    });
+}
+
 void DnsResolver::ResolveExternalInterfaceConstraintIndex() noexcept
 try
 {

src/windows/common/DnsResolver.h

@@ -43,6 +43,10 @@ class DnsResolver
         // Unique query id.
         uint32_t m_id{};
 
+        // Transaction ID from the original DNS request header (first 2 bytes, network byte order).
+        // Used to construct SERVFAIL responses when the Windows DNS API fails.
+        uint16_t m_dnsTransactionId{};
+
         // Callback to the parent object to notify about the DNS query completion.
         std::function<void(DnsQueryContext*, DNS_QUERY_RAW_RESULT*)> m_handleQueryCompletion;
 
@@ -78,6 +82,10 @@ class DnsResolver
     // queryResults - structure containing result of the DNS request.
     void HandleDnsQueryCompletion(_Inout_ DnsQueryContext* dnsQueryContext, _Inout_opt_ DNS_QUERY_RAW_RESULT* queryResults) noexcept;
 
+    // Build and send a minimal DNS SERVFAIL response (RFC 1035, RCODE=2) back to the Linux DNS client.
+    // This is used when the Windows DNS API fails, to prevent the Linux client from waiting until timeout.
+    void SendServfailResponse(uint16_t transactionId, const LX_GNS_DNS_CLIENT_IDENTIFIER& dnsClientIdentifier);
+
     void ResolveExternalInterfaceConstraintIndex() noexcept;
 
     // Callback that will be invoked by the DNS API whenever a request finishes. The callback is invoked on success, error or when request is cancelled.
@@ -105,7 +113,7 @@ class DnsResolver
     _Guarded_by_(m_dnsLock) uint32_t m_currentRequestId = 0;
 
     // Mapping request id to the request context structure.
-    _Guarded_by_(m_dnsLock) std::unordered_map<uint32_t, std::unique_ptr<DnsQueryContext>> m_dnsRequests {};
+    _Guarded_by_(m_dnsLock) std::unordered_map<uint32_t, std::unique_ptr<DnsQueryContext>> m_dnsRequests{};
 
     // Event that is set when all tracked DNS requests have completed.
     wil::unique_event m_allRequestsFinished{wil::EventOptions::ManualReset};