Skip to content

Commit f2ae79a

Browse files
RussKieRussKie
committed
Generate tests runsheet using Arcade SDK
1 parent e4bbe10 commit f2ae79a

File tree

8 files changed

+429
-14
lines changed

8 files changed

+429
-14
lines changed

‎.github/workflows/tests-runner.yml‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# Executes all the tests on all the platforms
2+
name: Tests
3+
4+
on:
5+
workflow_dispatch:
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: true
10+
11+
jobs:
12+
# Duplicated jobs so their dependencies are not blocked on both the
13+
# setup jobs
14+
15+
generate_tests_matrix:
16+
name: Generate test runsheet
17+
runs-on: windows-latest
18+
outputs:
19+
runsheet: ${{ steps.generate_tests_matrix.outputs.runsheet }}
20+
steps:
21+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
22+
23+
- name: Install toolsets
24+
shell: pwsh
25+
run: |
26+
mkdir ./artifacts/tmp -force | Out-Null
27+
'<Project />' | Out-File -FilePath ./artifacts/tmp/install-toolset.proj -Encoding utf8
28+
./build.cmd -restore -projects ./artifacts/tmp/install-toolset.proj
29+
30+
- name: Generate test runsheet
31+
id: generate_tests_matrix
32+
shell: pwsh
33+
run: |
34+
./build.cmd -test /p:TestRunnerName=TestRunsheetBuilder -bl -c Release
35+
36+
- name: Upload logs, and test results
37+
if: ${{ always() }}
38+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
39+
with:
40+
name: runsheet-logs
41+
path: |
42+
${{ github.workspace }}/artifacts/log/*/*.binlog
43+
${{ github.workspace }}/artifacts/log/*/TestLogs/**
44+
${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json
45+
retention-days: 3
46+
47+
generate_e2e_matrix:
48+
name: Generate E2E test runsheet
49+
runs-on: windows-latest
50+
outputs:
51+
runsheet: ${{ steps.generate_e2e_matrix.outputs.runsheet }}
52+
steps:
53+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
54+
55+
- name: Build with packages
56+
run: |
57+
./build.cmd -restore -build -pack -c Release -ci -bl /p:InstallBrowsersForPlaywright=false /p:SkipTestProjects=true /p:CI=false
58+
59+
- name: Generate test runsheet
60+
id: generate_e2e_matrix
61+
run: |
62+
./build.cmd -test /p:TestRunnerName=TestRunsheetBuilder /p:FullE2e=true -bl -c Release
63+
64+
- name: Upload built NuGets
65+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
66+
with:
67+
name: built-nugets
68+
path: artifacts/packages
69+
retention-days: 3
70+
71+
- name: Upload logs, and test results
72+
if: ${{ always() }}
73+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
74+
with:
75+
name: runsheet-e2e-logs
76+
path: |
77+
${{ github.workspace }}/artifacts/log/*/*.binlog
78+
${{ github.workspace }}/artifacts/log/*/TestLogs/**
79+
${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json
80+
retention-days: 3
81+
82+
run_tests:
83+
name: Test
84+
needs: generate_tests_matrix
85+
strategy:
86+
fail-fast: false
87+
matrix:
88+
tests: ${{ fromJson(needs.generate_tests_matrix.outputs.runsheet) }}
89+
90+
runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix
91+
92+
steps:
93+
- name: Trust HTTPS development certificate
94+
if: runner.os == 'Linux'
95+
run: dotnet dev-certs https --trust
96+
97+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
98+
99+
- name: Verify Docker is running
100+
# nested docker containers not supported on windows
101+
if: runner.os == 'Linux'
102+
run: docker info
103+
104+
- name: Install Azure Functions Core Tools
105+
if: runner.os == 'Linux'
106+
run: |
107+
sudo apt-get update
108+
sudo apt-get install -y azure-functions-core-tools-4
109+
110+
- name: Test ${{ matrix.tests.project }}
111+
run: |
112+
${{ matrix.tests.command }}
113+
114+
- name: Upload test results
115+
if: always()
116+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
117+
with:
118+
name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs
119+
path: |
120+
${{ github.workspace }}/artifacts/TestResults/*/*.trx
121+
${{ github.workspace }}/artifacts/log/*/TestLogs/**
122+
retention-days: 3
123+
124+
- name: Upload logs
125+
if: failure()
126+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
127+
with:
128+
name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-binlogs
129+
path: |
130+
${{ github.workspace }}/artifacts/log/*/*.binlog
131+
retention-days: 3
132+
133+
run_e2e_tests:
134+
name: E2ETest
135+
needs: generate_e2e_matrix
136+
strategy:
137+
fail-fast: false
138+
matrix:
139+
tests: ${{ fromJson(needs.generate_e2e_matrix.outputs.runsheet) }}
140+
141+
runs-on: ${{ matrix.tests.os }} # Use the OS from the matrix
142+
143+
steps:
144+
- name: Trust HTTPS development certificate
145+
if: runner.os == 'Linux'
146+
run: dotnet dev-certs https --trust
147+
148+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
149+
150+
- name: Download built NuGets
151+
uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
152+
with:
153+
pattern: built-nugets
154+
path: ${{ github.workspace }}/artifacts/packages
155+
156+
- name: Copy NuGets to the correct location
157+
shell: pwsh
158+
run:
159+
Move-Item -Path "${{ github.workspace }}/artifacts/packages/built-nugets/Release" -Destination "${{ github.workspace }}/artifacts/packages"
160+
161+
- name: Verify Docker is running
162+
# nested docker containers not supported on windows
163+
if: runner.os == 'Linux'
164+
run: docker info
165+
166+
- name: Install Azure Functions Core Tools
167+
if: runner.os == 'Linux'
168+
run: |
169+
sudo apt-get update
170+
sudo apt-get install -y azure-functions-core-tools-4
171+
172+
- name: Test ${{ matrix.tests.project }}
173+
env:
174+
BUILT_NUGETS_PATH: ${{ github.workspace }}/artifacts/packages/Release/Shipping
175+
run: |
176+
${{ matrix.tests.command }}
177+
178+
- name: Upload test results
179+
if: always()
180+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
181+
with:
182+
name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-logs
183+
path: |
184+
${{ github.workspace }}/artifacts/TestResults/*/*.trx
185+
${{ github.workspace }}/artifacts/log/*/TestLogs/**
186+
retention-days: 3
187+
188+
- name: Upload logs
189+
if: failure()
190+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
191+
with:
192+
name: ${{ matrix.tests.project }}-${{ matrix.tests.os }}-binlogs
193+
path: |
194+
${{ github.workspace }}/artifacts/log/*/*.binlog
195+
retention-days: 3

‎eng/AfterSolutionBuild.targets‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
```
5151
5252
-->
53-
<Target Name="_GenerateTestMatrix" BeforeTargets="Test" Condition=" '$(TestRunnerName)' == 'QuarantinedTestRunsheetBuilder' ">
53+
<Target Name="_GenerateTestMatrix" BeforeTargets="Test" Condition=" '$(TestRunnerName)' == 'TestRunsheetBuilder' or '$(TestRunnerName)' == 'QuarantinedTestRunsheetBuilder' ">
5454
<PropertyGroup>
5555
<_CombinedRunsheetFile>$(ArtifactsTmpDir)/combined_runsheet.json</_CombinedRunsheetFile>
5656
<_Command>
@@ -65,7 +65,9 @@
6565
$combined += @($content)
6666
}
6767
}
68-
$jsonString = ($combined | ConvertTo-Json -Depth 10 -Compress)
68+
69+
# See https://collectingwisdom.com/powershell-convertto-json-single-item-array/
70+
$jsonString = (ConvertTo-Json $combined -Depth 10 -Compress)
6971
$jsonString | Set-Content '$(_CombinedRunsheetFile)';
7072

7173
# determine if the script is running in a GitHub Actions environment

‎eng/TestRunsheetBuilder/TestRunsheetBuilder.targets‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
<Project>
2+
<!--
3+
4+
This file is used to generate a list of tests to run on GitHub Actions.
5+
6+
- For projects not requiring packages:
7+
.\build.cmd -test /p:TestRunnerName=TestRunsheetBuilder [/bl /p:GITHUB_ACTIONS=true]
8+
9+
- For projects requiring packages (e.g., E2E, templates and playground):
10+
.\build.cmd -test /p:TestRunnerName=TestRunsheetBuilder /p:FullE2e=true [/bl /p:GITHUB_ACTIONS=true]
11+
12+
For the large part this is a copy of the Arcade SDK's implementations:
13+
- https://github.com/dotnet/arcade/blob/b888df17/src/Microsoft.DotNet.Arcade.Sdk/tools/XUnit/XUnit.Runner.targets
14+
- https://github.com/dotnet/arcade/blob/b888df17/src/Microsoft.DotNet.Arcade.Sdk/tools/VSTest.targets
15+
-->
16+
17+
<Target Name="RunTests"
18+
Outputs="%(TestToRun.ResultsStdOutPath)"
19+
Condition=" '$(SkipTests)' != 'true' and '$(IsGitHubActionsRunner)' == 'true' and '$(RunOnGithubActions)' == 'true' ">
20+
21+
<PropertyGroup>
22+
<!--
23+
24+
The runsheet is created in two scenarios:
25+
- For projects requiring packages, only if full end-to-end tests (`FullE2e`) are enabled.
26+
- For projects not requiring packages, only if full end-to-end tests (`FullE2e`) are disabled.
27+
28+
This logic ensures runsheets are generated appropriately based on the project's characteristics and the testing scenario.
29+
30+
FIXME: Aspire.Playground.Tests are currently not running in "out of repo" mode.
31+
FIXME: Aspire.Templates.Tests are currently left alone.
32+
33+
-->
34+
<_RequiresPackages Condition=" '$(MSBuildProjectName)' == 'Aspire.EndToEnd.Tests' ">true</_RequiresPackages>
35+
36+
<_CreateRunsheet>false</_CreateRunsheet>
37+
<_CreateRunsheet Condition=" '$(_RequiresPackages)' == 'true' and '$(FullE2e)' == 'true' ">true</_CreateRunsheet>
38+
<_CreateRunsheet Condition=" '$(_RequiresPackages)' != 'true' and '$(FullE2e)' != 'true' ">true</_CreateRunsheet>
39+
40+
<_CreateRunsheet Condition=" '$(MSBuildProjectName)' == 'Aspire.Templates.Tests' ">false</_CreateRunsheet>
41+
</PropertyGroup>
42+
43+
<PropertyGroup>
44+
<!--
45+
We do not care whether the project is multi-targeting, we're only generating a command to kick off the testing sequence,
46+
which in turn will run the tests for all the target frameworks.
47+
48+
So, instead of using "%(TestToRun.ResultsFilePathWithoutExtension)" (which looks something like "Aspire.Cli.Tests_net8.0_x64")
49+
we use the project name (which looks something like "Aspire.Cli.Tests").
50+
-->
51+
<_TestRunsheet>$(MSBuildProjectName)</_TestRunsheet>
52+
<_TestRunsheetFileNameWindows>$(ArtifactsTmpDir)\$(_TestRunsheet).win.runsheet.json</_TestRunsheetFileNameWindows>
53+
<_TestRunsheetFileNameLinux>$(ArtifactsTmpDir)\$(_TestRunsheet).linux.runsheet.json</_TestRunsheetFileNameLinux>
54+
55+
<_TestBinLog>$([MSBuild]::NormalizePath($(ArtifactsLogDir), '$(_TestRunsheet).binlog'))</_TestBinLog>
56+
57+
<_RelativeTestProjectPath>$([System.String]::Copy('$(MSBuildProjectFullPath)').Replace('$(RepoRoot)', '%24(pwd)/'))</_RelativeTestProjectPath>
58+
<_RelativeTestBinLog>$([System.String]::Copy('$(_TestBinLog)').Replace('$(RepoRoot)', '%24(pwd)/'))</_RelativeTestBinLog>
59+
60+
<_TestRunnerWindows>./eng/build.ps1</_TestRunnerWindows>
61+
<_TestRunnerLinux>./eng/build.sh</_TestRunnerLinux>
62+
<_TestCommand>-restore -build -test -projects &quot;$(_RelativeTestProjectPath)&quot; /bl:&quot;$(_RelativeTestBinLog)&quot; -c $(Configuration) -ci /p:CI=false</_TestCommand>
63+
64+
<!-- Tests requiring packages must be run in the "out of repo" mode -->
65+
<_TestCommand Condition=" '$(_RequiresPackages)' == 'true' ">$(_TestCommand) /p:TestsRunningOutsideOfRepo=true</_TestCommand>
66+
67+
<!-- Replace \ with /, and then escape " with \", so we have a compliant JSON -->
68+
<_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('&quot;', '\&quot;'))</_TestCommand>
69+
70+
<_TestRunsheetWindows>{ "project": "$(_TestRunsheet)", "os": "windows-latest", "command": "./eng/build.ps1 $(_TestCommand)" }</_TestRunsheetWindows>
71+
<_TestRunsheetLinux>{ "project": "$(_TestRunsheet)", "os": "ubuntu-latest", "command": "./eng/build.sh $(_TestCommand)" }</_TestRunsheetLinux>
72+
</PropertyGroup>
73+
74+
<ItemGroup>
75+
<_OutputFiles Include="$(_TestRunsheetFileNameWindows)" />
76+
<_OutputFiles Include="$(_TestRunsheetFileNameLinux)" />
77+
</ItemGroup>
78+
79+
<MakeDir Directories="@(_OutputFiles->'%(RootDir)%(Directory)')"/>
80+
<Delete Files="@(_OutputFiles)" />
81+
82+
<WriteLinesToFile
83+
Condition=" '$(RunOnGithubActionsWindows)' == 'true' and '$(_CreateRunsheet)' == 'true' "
84+
File="$(_TestRunsheetFileNameWindows)"
85+
Lines="$(_TestRunsheetWindows)"
86+
Overwrite="true"
87+
WriteOnlyWhenDifferent="true" />
88+
<WriteLinesToFile
89+
Condition=" '$(RunOnGithubActionsLinux)' == 'true' and '$(_CreateRunsheet)' == 'true' "
90+
File="$(_TestRunsheetFileNameLinux)"
91+
Lines="$(_TestRunsheetLinux)"
92+
Overwrite="true"
93+
WriteOnlyWhenDifferent="true" />
94+
95+
<!--
96+
On Linux there's a bug in MSBuild, which "normalises" all slashes (see https://github.com/dotnet/msbuild/issues/3468).
97+
This is a workaround to replace `/"` with the required `\"`.
98+
-->
99+
<Exec Command="pwsh -Command &quot;(Get-Content -Path '$(_TestRunsheetFileNameWindows)') -replace '/\&quot;', '\\\&quot;' | Set-Content -Path '$(_TestRunsheetFileNameWindows)'&quot; "
100+
Condition=" Exists('$(_TestRunsheetFileNameWindows)') and '$(BuildOs)' != 'windows' " />
101+
<Exec Command="pwsh -Command &quot;(Get-Content -Path '$(_TestRunsheetFileNameLinux)') -replace '/\&quot;', '\\\&quot;' | Set-Content -Path '$(_TestRunsheetFileNameLinux)'&quot; "
102+
Condition=" Exists('$(_TestRunsheetFileNameLinux)') and '$(BuildOs)' != 'windows' " />
103+
104+
<!--
105+
The final piece of the puzzle is in eng/AfterSolutionBuild.targets, where we combine the runsheets from all the test projects into a single runsheet.
106+
-->
107+
</Target>
108+
109+
</Project>

‎eng/Testing.targets‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,7 @@
5858
<SkipTests>true</SkipTests>
5959

6060
<!-- Only run tests if the build is running on GitHub Actions -->
61-
<SkipTests Condition=" '$(IsGitHubActionsRunner)' == 'true' and '$(BuildOs)' == 'windows' and '$(RunOnGithubActionsWindows)' == 'true' ">false</SkipTests>
62-
<SkipTests Condition=" '$(IsGitHubActionsRunner)' == 'true' and '$(BuildOs)' != 'windows' and '$(RunOnGithubActionsLinux)' == 'true' ">false</SkipTests>
61+
<SkipTests Condition=" '$(IsGitHubActionsRunner)' == 'true' and '$(RunOnGithubActions)' == 'true' ">false</SkipTests>
6362

6463
<!-- Only run tests if the build is running on Helix infra -->
6564
<SkipTests Condition=" '$(IsAzdoHelixRunner)' == 'true' and '$(RunOnAzdoHelix)' == 'true' ">false</SkipTests>

‎eng/Xunit3/Microsoft.Testing.Platform.targets‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,14 @@
7878
<_ResultsFileToDisplay Condition="!Exists('$(_ResultsFileToDisplay)')">%(TestToRun.ResultsStdOutPath)</_ResultsFileToDisplay>
7979
</PropertyGroup>
8080

81+
<!-- Generate a test report -->
82+
<Exec Command="pwsh -command &quot;$(RepoRoot)eng/scripts/gha-testreport.ps1 -TestResultsFolder '$(_TestResultDirectory)' -TestSummaryOutputFolder '$(TestResultsLogDir)'&quot;"
83+
Condition=" '$(IsGitHubActionsRunner)' == 'true' "
84+
WorkingDirectory="$(RepoRoot)"
85+
IgnoreExitCode="true"
86+
ContinueOnError="WarnAndContinue"
87+
/>
88+
8189
<!--
8290
Ideally we would set ContinueOnError="ErrorAndContinue" so that when a test fails in multi-targeted test project
8391
we'll still run tests for all target frameworks. ErrorAndContinue doesn't work well on Linux though: https://github.com/Microsoft/msbuild/issues/3961.

0 commit comments

Comments
 (0)