diff --git a/.github/workflows/tests-runner.yml b/.github/workflows/tests-runner.yml new file mode 100644 index 00000000000..fe18e832669 --- /dev/null +++ b/.github/workflows/tests-runner.yml @@ -0,0 +1,208 @@ +# Executes all the tests on all the platforms +name: Tests + +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Duplicated jobs so their dependencies are not blocked on both the + # setup jobs + + # Generates a runsheet for all the tests in the solution that do not require + # NuGet packages to be built. + # The runsheet generation is expected to be fast. + generate_tests_matrix: + name: Generate test runsheet + runs-on: windows-latest + outputs: + runsheet: ${{ steps.generate_tests_matrix.outputs.runsheet }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + # In order to build the solution, we need to install the SDK and the toolsets first + # as defined in global.json. For this, create a temporary project file and run the + # build command with the -restore option. This will install the SDK and toolsets. + # + # We don't want to run 'build.cmd -restore' as it will also restore all the packages, + # which takes a long time and is not needed for this job. + - name: Install toolsets + shell: pwsh + run: | + mkdir ./artifacts/tmp -force | Out-Null + '' | Out-File -FilePath ./artifacts/tmp/install-toolset.proj -Encoding utf8 + ./build.cmd -restore -projects ./artifacts/tmp/install-toolset.proj + + - name: Generate test runsheet + id: generate_tests_matrix + shell: pwsh + run: | + ./build.cmd -test /p:TestRunnerName=TestRunsheetBuilder -bl -c Release + + - name: Upload logs, and test results + if: ${{ always() }} + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: runsheet-logs + path: | + ${{ github.workspace }}/artifacts/log/*/*.binlog + ${{ github.workspace }}/artifacts/log/*/TestLogs/** + ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json + retention-days: 3 + + # Generates a runsheet for all the tests in the solution that DO require + # NuGet packages to be built. + # The runsheet generation is expected to be slow as we need to restore and build + # the whole solution and publish all the packages. + generate_e2e_matrix: + name: Generate E2E test runsheet + runs-on: windows-latest + outputs: + runsheet: ${{ steps.generate_e2e_matrix.outputs.runsheet }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Build with packages + run: | + ./build.cmd -restore -build -pack -c Release -ci -bl /p:InstallBrowsersForPlaywright=false /p:SkipTestProjects=true /p:CI=false + + - name: Generate test runsheet + id: generate_e2e_matrix + run: | + ./build.cmd -test /p:TestRunnerName=TestRunsheetBuilder /p:FullE2e=true -bl -c Release + + - name: Upload built NuGets + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: built-nugets + path: artifacts/packages + retention-days: 3 + + - name: Upload logs, and test results + if: ${{ always() }} + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: runsheet-e2e-logs + path: | + ${{ github.workspace }}/artifacts/log/*/*.binlog + ${{ github.workspace }}/artifacts/log/*/TestLogs/** + ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json + retention-days: 3 + + run_tests: + name: Test + needs: generate_tests_matrix + strategy: + fail-fast: false + matrix: + tests: ${{ fromJson(needs.generate_tests_matrix.outputs.runsheet) }} + + runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix + + steps: + - name: Trust HTTPS development certificate + if: runner.os == 'Linux' + run: dotnet dev-certs https --trust + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Verify Docker is running + # nested docker containers not supported on windows + if: runner.os == 'Linux' + run: docker info + + - name: Install Azure Functions Core Tools + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y azure-functions-core-tools-4 + + - name: Test ${{ matrix.tests.project }} + run: | + ${{ matrix.tests.command }} + + - name: Upload test results + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs + path: | + ${{ github.workspace }}/artifacts/TestResults/*/*.trx + ${{ github.workspace }}/artifacts/log/*/TestLogs/** + retention-days: 30 + + - name: Upload logs + if: failure() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-binlogs + path: | + ${{ github.workspace }}/artifacts/log/*/*.binlog + retention-days: 3 + + run_e2e_tests: + name: E2ETest + needs: generate_e2e_matrix + strategy: + fail-fast: false + matrix: + tests: ${{ fromJson(needs.generate_e2e_matrix.outputs.runsheet) }} + + runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix + + steps: + - name: Trust HTTPS development certificate + if: runner.os == 'Linux' + run: dotnet dev-certs https --trust + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download built NuGets + uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9 + with: + pattern: built-nugets + path: ${{ github.workspace }}/artifacts/packages + + - name: Copy NuGets to the correct location + shell: pwsh + run: + Move-Item -Path "${{ github.workspace }}/artifacts/packages/built-nugets/Release" -Destination "${{ github.workspace }}/artifacts/packages" + + - name: Verify Docker is running + # nested docker containers not supported on windows + if: runner.os == 'Linux' + run: docker info + + - name: Install Azure Functions Core Tools + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y azure-functions-core-tools-4 + + - name: Test ${{ matrix.tests.project }} + env: + BUILT_NUGETS_PATH: ${{ github.workspace }}/artifacts/packages/Release/Shipping + run: | + ${{ matrix.tests.command }} + + - name: Upload test results + if: always() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs + path: | + ${{ github.workspace }}/artifacts/TestResults/*/*.trx + ${{ github.workspace }}/artifacts/log/*/TestLogs/** + retention-days: 30 + + - name: Upload logs + if: failure() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-binlogs + path: | + ${{ github.workspace }}/artifacts/log/*/*.binlog + retention-days: 3 diff --git a/eng/AfterSolutionBuild.targets b/eng/AfterSolutionBuild.targets index cadc5b7fbf3..c99de717e62 100644 --- a/eng/AfterSolutionBuild.targets +++ b/eng/AfterSolutionBuild.targets @@ -50,7 +50,7 @@ ``` --> - + <_CombinedRunsheetFile>$(ArtifactsTmpDir)/combined_runsheet.json <_Command> @@ -65,7 +65,9 @@ $combined += @($content) } } - $jsonString = ($combined | ConvertTo-Json -Depth 10 -Compress) + + # See https://collectingwisdom.com/powershell-convertto-json-single-item-array/ + $jsonString = (ConvertTo-Json $combined -Depth 10 -Compress) $jsonString | Set-Content '$(_CombinedRunsheetFile)'; # determine if the script is running in a GitHub Actions environment diff --git a/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets b/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets new file mode 100644 index 00000000000..484719d23f9 --- /dev/null +++ b/eng/TestRunsheetBuilder/TestRunsheetBuilder.targets @@ -0,0 +1,109 @@ + + + + + + + + <_RequiresPackages Condition=" '$(MSBuildProjectName)' == 'Aspire.EndToEnd.Tests' ">true + + <_CreateRunsheet>false + <_CreateRunsheet Condition=" '$(_RequiresPackages)' == 'true' and '$(FullE2e)' == 'true' ">true + <_CreateRunsheet Condition=" '$(_RequiresPackages)' != 'true' and '$(FullE2e)' != 'true' ">true + + <_CreateRunsheet Condition=" '$(MSBuildProjectName)' == 'Aspire.Templates.Tests' ">false + + + + + <_TestRunsheet>$(MSBuildProjectName) + <_TestRunsheetFileNameWindows>$(ArtifactsTmpDir)\$(_TestRunsheet).win.runsheet.json + <_TestRunsheetFileNameLinux>$(ArtifactsTmpDir)\$(_TestRunsheet).linux.runsheet.json + + <_TestBinLog>$([MSBuild]::NormalizePath($(ArtifactsLogDir), '$(_TestRunsheet).binlog')) + + <_RelativeTestProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '%24(pwd)/')) + <_RelativeTestBinLog>$([System.String]::Copy('$(_TestBinLog)').Replace('$(RepoRoot)', '%24(pwd)/')) + + <_TestRunnerWindows>./eng/build.ps1 + <_TestRunnerLinux>./eng/build.sh + <_TestCommand>-restore -build -test -projects "$(_RelativeTestProjectPath)" /bl:"$(_RelativeTestBinLog)" -c $(Configuration) -ci /p:CI=false + + + <_TestCommand Condition=" '$(_RequiresPackages)' == 'true' ">$(_TestCommand) /p:TestsRunningOutsideOfRepo=true + + + <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('"', '\"')) + + <_TestRunsheetWindows>{ "project": "$(_TestRunsheet)", "os": "windows-latest", "command": "./eng/build.ps1 $(_TestCommand)" } + <_TestRunsheetLinux>{ "project": "$(_TestRunsheet)", "os": "ubuntu-latest", "command": "./eng/build.sh $(_TestCommand)" } + + + + <_OutputFiles Include="$(_TestRunsheetFileNameWindows)" /> + <_OutputFiles Include="$(_TestRunsheetFileNameLinux)" /> + + + + + + + + + + + + + + + + diff --git a/eng/Testing.targets b/eng/Testing.targets index bda77e5bd73..b631ca7b509 100644 --- a/eng/Testing.targets +++ b/eng/Testing.targets @@ -58,8 +58,7 @@ true - false - false + false false diff --git a/eng/Xunit3/Microsoft.Testing.Platform.targets b/eng/Xunit3/Microsoft.Testing.Platform.targets index 0934af6a81c..ad220b02f32 100644 --- a/eng/Xunit3/Microsoft.Testing.Platform.targets +++ b/eng/Xunit3/Microsoft.Testing.Platform.targets @@ -78,6 +78,14 @@ <_ResultsFileToDisplay Condition="!Exists('$(_ResultsFileToDisplay)')">%(TestToRun.ResultsStdOutPath) + + +