diff --git a/.github/workflows/tests-runner.yml b/.github/workflows/tests-runner.yml
new file mode 100644
index 00000000000..fe18e832669
--- /dev/null
+++ b/.github/workflows/tests-runner.yml
@@ -0,0 +1,208 @@
+# Executes all the tests on all the platforms
+name: Tests
+
+on:
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ # Duplicated jobs so their dependencies are not blocked on both the
+ # setup jobs
+
+ # Generates a runsheet for all the tests in the solution that do not require
+ # NuGet packages to be built.
+ # The runsheet generation is expected to be fast.
+ generate_tests_matrix:
+ name: Generate test runsheet
+ runs-on: windows-latest
+ outputs:
+ runsheet: ${{ steps.generate_tests_matrix.outputs.runsheet }}
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+ # In order to build the solution, we need to install the SDK and the toolsets first
+ # as defined in global.json. For this, create a temporary project file and run the
+ # build command with the -restore option. This will install the SDK and toolsets.
+ #
+ # We don't want to run 'build.cmd -restore' as it will also restore all the packages,
+ # which takes a long time and is not needed for this job.
+ - name: Install toolsets
+ shell: pwsh
+ run: |
+ mkdir ./artifacts/tmp -force | Out-Null
+ '' | Out-File -FilePath ./artifacts/tmp/install-toolset.proj -Encoding utf8
+ ./build.cmd -restore -projects ./artifacts/tmp/install-toolset.proj
+
+ - name: Generate test runsheet
+ id: generate_tests_matrix
+ shell: pwsh
+ run: |
+ ./build.cmd -test /p:TestRunnerName=TestRunsheetBuilder -bl -c Release
+
+ - name: Upload logs, and test results
+ if: ${{ always() }}
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: runsheet-logs
+ path: |
+ ${{ github.workspace }}/artifacts/log/*/*.binlog
+ ${{ github.workspace }}/artifacts/log/*/TestLogs/**
+ ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json
+ retention-days: 3
+
+ # Generates a runsheet for all the tests in the solution that DO require
+ # NuGet packages to be built.
+ # The runsheet generation is expected to be slow as we need to restore and build
+ # the whole solution and publish all the packages.
+ generate_e2e_matrix:
+ name: Generate E2E test runsheet
+ runs-on: windows-latest
+ outputs:
+ runsheet: ${{ steps.generate_e2e_matrix.outputs.runsheet }}
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+ - name: Build with packages
+ run: |
+ ./build.cmd -restore -build -pack -c Release -ci -bl /p:InstallBrowsersForPlaywright=false /p:SkipTestProjects=true /p:CI=false
+
+ - name: Generate test runsheet
+ id: generate_e2e_matrix
+ run: |
+ ./build.cmd -test /p:TestRunnerName=TestRunsheetBuilder /p:FullE2e=true -bl -c Release
+
+ - name: Upload built NuGets
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: built-nugets
+ path: artifacts/packages
+ retention-days: 3
+
+ - name: Upload logs, and test results
+ if: ${{ always() }}
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: runsheet-e2e-logs
+ path: |
+ ${{ github.workspace }}/artifacts/log/*/*.binlog
+ ${{ github.workspace }}/artifacts/log/*/TestLogs/**
+ ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json
+ retention-days: 3
+
+ run_tests:
+ name: Test
+ needs: generate_tests_matrix
+ strategy:
+ fail-fast: false
+ matrix:
+ tests: ${{ fromJson(needs.generate_tests_matrix.outputs.runsheet) }}
+
+ runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix
+
+ steps:
+ - name: Trust HTTPS development certificate
+ if: runner.os == 'Linux'
+ run: dotnet dev-certs https --trust
+
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+ - name: Verify Docker is running
+ # nested docker containers not supported on windows
+ if: runner.os == 'Linux'
+ run: docker info
+
+ - name: Install Azure Functions Core Tools
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y azure-functions-core-tools-4
+
+ - name: Test ${{ matrix.tests.project }}
+ run: |
+ ${{ matrix.tests.command }}
+
+ - name: Upload test results
+ if: always()
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs
+ path: |
+ ${{ github.workspace }}/artifacts/TestResults/*/*.trx
+ ${{ github.workspace }}/artifacts/log/*/TestLogs/**
+ retention-days: 30
+
+ - name: Upload logs
+ if: failure()
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-binlogs
+ path: |
+ ${{ github.workspace }}/artifacts/log/*/*.binlog
+ retention-days: 3
+
+ run_e2e_tests:
+ name: E2ETest
+ needs: generate_e2e_matrix
+ strategy:
+ fail-fast: false
+ matrix:
+ tests: ${{ fromJson(needs.generate_e2e_matrix.outputs.runsheet) }}
+
+ runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix
+
+ steps:
+ - name: Trust HTTPS development certificate
+ if: runner.os == 'Linux'
+ run: dotnet dev-certs https --trust
+
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+ - name: Download built NuGets
+ uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
+ with:
+ pattern: built-nugets
+ path: ${{ github.workspace }}/artifacts/packages
+
+ - name: Copy NuGets to the correct location
+ shell: pwsh
+ run:
+ Move-Item -Path "${{ github.workspace }}/artifacts/packages/built-nugets/Release" -Destination "${{ github.workspace }}/artifacts/packages"
+
+ - name: Verify Docker is running
+ # nested docker containers not supported on windows
+ if: runner.os == 'Linux'
+ run: docker info
+
+ - name: Install Azure Functions Core Tools
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y azure-functions-core-tools-4
+
+ - name: Test ${{ matrix.tests.project }}
+ env:
+ BUILT_NUGETS_PATH: ${{ github.workspace }}/artifacts/packages/Release/Shipping
+ run: |
+ ${{ matrix.tests.command }}
+
+ - name: Upload test results
+ if: always()
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs
+ path: |
+ ${{ github.workspace }}/artifacts/TestResults/*/*.trx
+ ${{ github.workspace }}/artifacts/log/*/TestLogs/**
+ retention-days: 30
+
+ - name: Upload logs
+ if: failure()
+ uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+ with:
+ name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-binlogs
+ path: |
+ ${{ github.workspace }}/artifacts/log/*/*.binlog
+ retention-days: 3
diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets
index cadc5b7fbf3..c99de717e62 100644
--- a/eng/AfterSolutionBuild.targets
+++ b/eng/AfterSolutionBuild.targets
@@ -50,7 +50,7 @@
```
-->
-
+
<_CombinedRunsheetFile>$(ArtifactsTmpDir)/combined_runsheet.json
<_Command>
@@ -65,7 +65,9 @@
$combined += @($content)
}
}
- $jsonString = ($combined | ConvertTo-Json -Depth 10 -Compress)
+
+ # See https://collectingwisdom.com/powershell-convertto-json-single-item-array/
+ $jsonString = (ConvertTo-Json $combined -Depth 10 -Compress)
$jsonString | Set-Content '$(_CombinedRunsheetFile)';
# determine if the script is running in a GitHub Actions environment
diff --git a/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets b/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets
new file mode 100644
index 00000000000..484719d23f9
--- /dev/null
+++ b/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets
@@ -0,0 +1,109 @@
+
+
+
+
+
+
+
+ <_RequiresPackages Condition=" '$(MSBuildProjectName)' == 'Aspire.EndToEnd.Tests' ">true
+
+ <_CreateRunsheet>false
+ <_CreateRunsheet Condition=" '$(_RequiresPackages)' == 'true' and '$(FullE2e)' == 'true' ">true
+ <_CreateRunsheet Condition=" '$(_RequiresPackages)' != 'true' and '$(FullE2e)' != 'true' ">true
+
+ <_CreateRunsheet Condition=" '$(MSBuildProjectName)' == 'Aspire.Templates.Tests' ">false
+
+
+
+
+ <_TestRunsheet>$(MSBuildProjectName)
+ <_TestRunsheetFileNameWindows>$(ArtifactsTmpDir)\$(_TestRunsheet).win.runsheet.json
+ <_TestRunsheetFileNameLinux>$(ArtifactsTmpDir)\$(_TestRunsheet).linux.runsheet.json
+
+ <_TestBinLog>$([MSBuild]::NormalizePath($(ArtifactsLogDir), '$(_TestRunsheet).binlog'))
+
+ <_RelativeTestProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '%24(pwd)/'))
+ <_RelativeTestBinLog>$([System.String]::Copy('$(_TestBinLog)').Replace('$(RepoRoot)', '%24(pwd)/'))
+
+ <_TestRunnerWindows>./eng/build.ps1
+ <_TestRunnerLinux>./eng/build.sh
+ <_TestCommand>-restore -build -test -projects "$(_RelativeTestProjectPath)" /bl:"$(_RelativeTestBinLog)" -c $(Configuration) -ci /p:CI=false
+
+
+ <_TestCommand Condition=" '$(_RequiresPackages)' == 'true' ">$(_TestCommand) /p:TestsRunningOutsideOfRepo=true
+
+
+ <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('"', '\"'))
+
+ <_TestRunsheetWindows>{ "project": "$(_TestRunsheet)", "os": "windows-latest", "command": "./eng/build.ps1 $(_TestCommand)" }
+ <_TestRunsheetLinux>{ "project": "$(_TestRunsheet)", "os": "ubuntu-latest", "command": "./eng/build.sh $(_TestCommand)" }
+
+
+
+ <_OutputFiles Include="$(_TestRunsheetFileNameWindows)" />
+ <_OutputFiles Include="$(_TestRunsheetFileNameLinux)" />
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/eng/Testing.targets b/eng/Testing.targets
index bda77e5bd73..b631ca7b509 100644
--- a/eng/Testing.targets
+++ b/eng/Testing.targets
@@ -58,8 +58,7 @@
true
- false
- false
+ false
false
diff --git a/eng/Xunit3/Microsoft.Testing.Platform.targets b/eng/Xunit3/Microsoft.Testing.Platform.targets
index 0934af6a81c..ad220b02f32 100644
--- a/eng/Xunit3/Microsoft.Testing.Platform.targets
+++ b/eng/Xunit3/Microsoft.Testing.Platform.targets
@@ -78,6 +78,14 @@
<_ResultsFileToDisplay Condition="!Exists('$(_ResultsFileToDisplay)')">%(TestToRun.ResultsStdOutPath)
+
+
+