devicehost: stop re-signing and fix MSI installer failing to replace wsldevicehost.dll (#40075)

* devicehost: stop re-signing and fix MSI installer failing to replace
wsldevicehost.dll

* pr feedback

* use MSIRMSHUTDOWN = 1 instead of custom action

---------

Co-authored-by: Ben Hillis <benhill@ntdev.microsoft.com>

Author: benhillis

.pipelines/build-stage.yml

@@ -24,7 +24,7 @@ parameters:
     type: object
     default:
       - target: "wsl;libwsl;wslg;wslservice;wslhost;wslrelay;wslinstaller;wslinstall;initramfs;wslserviceproxystub;wslsettings;wslinstallerproxystub;testplugin"
-        pattern: "wsl.exe,libwsl.dll,wslg.exe,wslservice.exe,wslhost.exe,wslrelay.exe,wslinstaller.exe,wslinstall.dll,wslserviceproxystub.dll,wslsettings/wslsettings.dll,wslsettings/wslsettings.exe,wslinstallerproxystub.dll,wsldevicehost.dll,WSLDVCPlugin.dll,testplugin.dll,wsldeps.dll"
+        pattern: "wsl.exe,libwsl.dll,wslg.exe,wslservice.exe,wslhost.exe,wslrelay.exe,wslinstaller.exe,wslinstall.dll,wslserviceproxystub.dll,wslsettings/wslsettings.dll,wslsettings/wslsettings.exe,wslinstallerproxystub.dll,WSLDVCPlugin.dll,testplugin.dll,wsldeps.dll"
       - target: "msixgluepackage"
         pattern: "gluepackage.msix"
       - target: "msipackage"

CMakeLists.txt

@@ -157,7 +157,6 @@ set (GENERATED_DIR ${CMAKE_BINARY_DIR}/generated)
 file(MAKE_DIRECTORY ${GENERATED_DIR})
 
 set(PACKAGE_CERTIFICATE ${GENERATED_DIR}/dev-cert.pfx)
-file(CREATE_LINK ${WSL_DEVICE_HOST_SOURCE_DIR}/bin/${TARGET_PLATFORM}/wsldevicehost.dll ${BIN}/wsldevicehost.dll)
 file(CREATE_LINK ${WSLG_SOURCE_DIR}/${TARGET_PLATFORM}/${WSLG_TS_PLUGIN_DLL} ${BIN}/${WSLG_TS_PLUGIN_DLL})
 file(CREATE_LINK ${WSLDEPS_SOURCE_DIR}/bin/wsldeps.dll ${BIN}/wsldeps.dll)
 

msipackage/package.wix.in

@@ -221,6 +221,8 @@
                      N.B. It shouldn't be started on install because it will fail to start if the OC is missing, 
                      which would fail the install. -->
                 <ServiceControl Id="StopService" Stop="both" Remove="uninstall" Name="WSLService" Wait="yes" />
+
+                <File Id="wsldevicehost.dll" Source="${WSL_DEVICE_HOST_SOURCE_DIR}/bin/${TARGET_PLATFORM}/wsldevicehost.dll" />
             </Component>
 
             <Component Id="wslg" Guid="F0C8D6BA-1502-41E7-BF72-D93DFA134731" UninstallWhenSuperseded="yes" DisableRegistryReflection="yes" Bitness="always64">
@@ -232,7 +234,6 @@
                     <File Id="rdpnanoTransport.dll" Source="${MSRDC_SOURCE_DIR}/${TARGET_PLATFORM}/rdpnanoTransport.dll" />
                     <File Id="RdpWinStlHelper.dll" Source="${MSRDC_SOURCE_DIR}/${TARGET_PLATFORM}/RdpWinStlHelper.dll" />
                 <?endif?>
-                <File Id="wsldevicehost.dll" Source="${PACKAGE_INPUT_DIR}/wsldevicehost.dll" />
                 <File Id="${WSLG_TS_PLUGIN_DLL}" Source="${PACKAGE_INPUT_DIR}/${WSLG_TS_PLUGIN_DLL}" />
 
                 <!-- MSRDC Plugin registration -->
@@ -551,7 +552,7 @@
         <!-- Force all applications to exit during upgrade.
              See: https://learn.microsoft.com/en-us/windows/win32/msi/msirmshutdown
         -->
-        <Property Id="MSIRMSHUTDOWN" Value="0" Secure="yes" />
+        <Property Id="MSIRMSHUTDOWN" Value="1" Secure="yes" />
     </Package>
 </Wix>
 

packages.config

@@ -18,7 +18,7 @@
   <package id="Microsoft.WSL.bsdtar" version="0.0.2-2" />
   <package id="Microsoft.WSL.Dependencies.amd64fre" version="10.0.27820.1000-250318-1700.rs-base2-hyp" targetFramework="native" />
   <package id="Microsoft.WSL.Dependencies.arm64fre" version="10.0.27820.1000-250318-1700.rs-base2-hyp" targetFramework="native" />
-  <package id="Microsoft.WSL.DeviceHost" version="1.1.48-0" />
+  <package id="Microsoft.WSL.DeviceHost" version="1.2.10-0" />
   <package id="Microsoft.WSL.Kernel" version="6.6.114.1-1" targetFramework="native" />
   <package id="Microsoft.WSL.LinuxSdk" version="1.20.0" targetFramework="native" />
   <package id="Microsoft.WSL.TestDistro" version="2.7.1-1" />

test/windows/InstallerTests.cpp

@@ -175,8 +175,17 @@ class InstallerTests
         return wsl::windows::common::registry::ReadString(m_lxssKey.get(), L"MSI", L"ProductCode", L"");
     }
 
+    // Release any in-process COM DLLs (e.g. wslserviceproxystub.dll loaded by prior tests)
+    // so the Restart Manager doesn't detect the test process as holding files.
+    // This avoids install failures on older Server SKUs where the RM has stricter silent-mode behavior.
+    static void PrepareForMsiOperation()
+    {
+        CoFreeUnusedLibrariesEx(0, 0);
+    }
+
     void UninstallMsi()
     {
+        PrepareForMsiOperation();
         auto productCode = GetMsiProductCode();
         VERIFY_IS_FALSE(productCode.empty());
 
@@ -185,6 +194,7 @@ class InstallerTests
 
     void InstallMsi()
     {
+        PrepareForMsiOperation();
         CallMsiExec(std::format(L"/qn /norestart /i {} /L*V {}", m_msiPath, GenerateMsiLogPath()));
     }
 
@@ -367,6 +377,7 @@ class InstallerTests
         LogInfo("Installing: %ls", installerFile.c_str());
         if (wsl::shared::string::EndsWith<wchar_t>(installerFile, L".msi"))
         {
+            PrepareForMsiOperation();
             CallMsiExec(std::format(L"/qn /norestart /i {} /L*V {}", installerFile, GenerateMsiLogPath()));
         }
         else