From 56c5741e3d8808d81cd4540fd7fa5822830c47bb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:41:22 +0000 Subject: [PATCH 1/6] Initial plan From ecf5105d42c81b1ba0065718d1ea8c2d82099258 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:43:04 +0000 Subject: [PATCH 2/6] Remove screenshot-test-comment workflow, add CI error step to ScreenshotTests job --- .github/workflows/main.yml | 4 + .github/workflows/screenshot-test-comment.yml | 138 ------------------ 2 files changed, 4 insertions(+), 138 deletions(-) delete mode 100644 .github/workflows/screenshot-test-comment.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3f986c9722..68f1840d07 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -163,6 +163,10 @@ jobs: - name: Test with Gradle Wrapper run: | ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-desktop:screenshotTest + - name: Report Screenshot Test failure + if: failure() + run: | + echo "::error title=Screenshot tests failed::Screenshot tests have failed. Download the 'screenshot-test-report' artifact from this run, then open jme3-screenshot-tests/build/reports/ScreenshotDiffReport.html to review the differences. If you did not intend to change any visuals, fix your changes so the tests pass. If you intended to change visuals, review the images in jme3-screenshot-tests/build/changed-images and commit replacements to jme3-screenshot-tests/src/test/resources. See https://github.com/jMonkeyEngine/jmonkeyengine/blob/master/jme3-screenshot-tests/README.md for more information." - name: Upload Test Reports uses: actions/upload-artifact@v7.0.1 if: always() diff --git a/.github/workflows/screenshot-test-comment.yml b/.github/workflows/screenshot-test-comment.yml deleted file mode 100644 index a445615273..0000000000 --- a/.github/workflows/screenshot-test-comment.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: Screenshot Test PR Comment - -# This workflow is designed to safely comment on PRs from forks -# It uses pull_request_target which has higher permissions than pull_request -# Security note: This workflow does NOT check out or execute code from the PR -# It only monitors the status of the ScreenshotTests job and posts comments -# (If this commenting was done in the main worflow it would not have the permissions -# to create a comment) - -on: - pull_request_target: - types: [opened, synchronize, reopened] - -jobs: - monitor-screenshot-tests: - name: Monitor Screenshot Tests and Comment - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - pull-requests: write - contents: read - steps: - - name: Wait for GitHub to register the workflow run - run: sleep 120 - - - name: Wait for Screenshot Tests to complete - uses: lewagon/wait-on-check-action@9312864dfbc9fd208e9c0417843430751c042800 # v1.7.0 - with: - ref: ${{ github.event.pull_request.head.sha }} - check-name: 'Run Screenshot Tests' - repo-token: ${{ secrets.GITHUB_TOKEN }} - wait-interval: 10 - allowed-conclusions: success,skipped,failure - - name: Check Screenshot Tests status - id: check-status - uses: actions/github-script@v9.0.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { owner, repo } = context.repo; - const ref = '${{ github.event.pull_request.head.sha }}'; - - // Get workflow runs for the PR - const runs = await github.rest.actions.listWorkflowRunsForRepo({ - owner, - repo, - head_sha: ref - }); - - // Find the ScreenshotTests job - let screenshotTestRun = null; - let workflowRunId = null; - for (const run of runs.data.workflow_runs) { - if (run.name === 'Build jMonkeyEngine') { - workflowRunId = run.id; - const jobs = await github.rest.actions.listJobsForWorkflowRun({ - owner, - repo, - run_id: run.id - }); - - for (const job of jobs.data.jobs) { - if (job.name === 'Run Screenshot Tests') { - screenshotTestRun = job; - break; - } - } - - if (screenshotTestRun) break; - } - } - - if (!screenshotTestRun) { - console.log('Screenshot test job not found'); - return; - } - - core.setOutput('run_url', `https://github.com/${owner}/${repo}/actions/runs/${workflowRunId}`); - - // Check if the job failed - if (screenshotTestRun.conclusion === 'failure') { - // Now check if the changed images artifact exists - const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner, - repo, - run_id: workflowRunId - }); - - const artifact = artifacts.data.artifacts.find(a => a.name === 'screenshot-test-report'); - if (artifact) { - core.setOutput('failed', 'true'); - core.setOutput('artifact_url', `https://github.com/${owner}/${repo}/actions/runs/${workflowRunId}/artifacts/${artifact.id}`); - } else { - console.log('Job failed but no changed images were generated.'); - core.setOutput('failed', 'false'); - } - } else { - core.setOutput('failed', 'false'); - } - - name: Find Existing Comment - uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 - id: existingCommentId - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Screenshot tests have failed. - - - name: Comment on PR if tests fail - if: steps.check-status.outputs.failed == 'true' - uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - 🖼️ **Screenshot tests have failed.** - - The purpose of these tests is to ensure that changes introduced in this PR don't break visual features. They are visual unit tests. - - 📄 **Where to find the report:** - - **[Direct Download: screenshot-test-report](${{ steps.check-status.outputs.artifact_url }})** - - Alternatively, go to the [failed run summary](${{ steps.check-status.outputs.run_url }}) > Artifacts > screenshot-test-report - - Download the zip and open jme3-screenshot-tests/build/reports/ScreenshotDiffReport.html - - ⚠️ **If you didn't expect to change anything visual:** - Fix your changes so the screenshot tests pass. - - ✅ **If you did mean to change things:** - Review the replacement images in jme3-screenshot-tests/build/changed-images to make sure they really are improvements and then replace and commit the replacement images at jme3-screenshot-tests/src/test/resources. - - ✨ **If you are creating entirely new tests:** - Find the new images in jme3-screenshot-tests/build/changed-images and commit the new images at jme3-screenshot-tests/src/test/resources. - - **Note;** it is very important that the committed reference images are created on the build pipeline, locally created images are not reliable. Similarly tests will fail locally but you can look at the report to check they are "visually similar". - - See https://github.com/jMonkeyEngine/jmonkeyengine/blob/master/jme3-screenshot-tests/README.md for more information - - Contact %40richardTingle (aka richtea) for guidance if required - edit-mode: replace - comment-id: ${{ steps.existingCommentId.outputs.comment-id }} From c7397c64e45505f0a8ba53dc3e82249de5b2a185 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:43:35 +0000 Subject: [PATCH 3/6] Improve readability of screenshot test failure error message --- .github/workflows/main.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68f1840d07..484dcc177c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -166,7 +166,14 @@ jobs: - name: Report Screenshot Test failure if: failure() run: | - echo "::error title=Screenshot tests failed::Screenshot tests have failed. Download the 'screenshot-test-report' artifact from this run, then open jme3-screenshot-tests/build/reports/ScreenshotDiffReport.html to review the differences. If you did not intend to change any visuals, fix your changes so the tests pass. If you intended to change visuals, review the images in jme3-screenshot-tests/build/changed-images and commit replacements to jme3-screenshot-tests/src/test/resources. See https://github.com/jMonkeyEngine/jmonkeyengine/blob/master/jme3-screenshot-tests/README.md for more information." + echo "::error title=Screenshot tests failed::Screenshot tests have failed.%0A%0A"\ + "To review the differences:%0A"\ + " 1. Download the 'screenshot-test-report' artifact from this run.%0A"\ + " 2. Open jme3-screenshot-tests/build/reports/ScreenshotDiffReport.html%0A%0A"\ + "If you did not intend to change any visuals: fix your changes so the tests pass.%0A"\ + "If you intended to change visuals: review jme3-screenshot-tests/build/changed-images"\ + " and commit replacements to jme3-screenshot-tests/src/test/resources.%0A%0A"\ + "See https://github.com/jMonkeyEngine/jmonkeyengine/blob/master/jme3-screenshot-tests/README.md for more information." - name: Upload Test Reports uses: actions/upload-artifact@v7.0.1 if: always() From 1e07709b39233f5a93de595bf38157a6a9d53911 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:49:23 +0000 Subject: [PATCH 4/6] Retry screenshot tests up to 3 times before failing --- .github/workflows/main.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 484dcc177c..f8be19ead1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -162,7 +162,17 @@ jobs: uses: gradle/actions/wrapper-validation@v6.1.0 - name: Test with Gradle Wrapper run: | - ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-desktop:screenshotTest + for attempt in 1 2 3; do + echo "Screenshot test attempt $attempt of 3" + if ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-desktop:screenshotTest; then + exit 0 + fi + if [ $attempt -lt 3 ]; then + echo "Attempt $attempt failed, retrying..." + fi + done + echo "All 3 attempts failed." + exit 1 - name: Report Screenshot Test failure if: failure() run: | From f7332e8ab9355fe53132769af2ffc1b2132e5173 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:06:48 +0000 Subject: [PATCH 5/6] Retry Android screenshot tests up to 3 times before failing --- .github/workflows/android-screenshot.yml | 40 +++++++++++++++--------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/android-screenshot.yml b/.github/workflows/android-screenshot.yml index 16d33203bb..c39fd0faf4 100644 --- a/.github/workflows/android-screenshot.yml +++ b/.github/workflows/android-screenshot.yml @@ -15,7 +15,7 @@ on: jobs: screenshot-test-android: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 60 steps: - name: checkout uses: actions/checkout@v6.0.2 @@ -54,20 +54,30 @@ jobs: profile: pixel_tablet emulator-options: -no-window -gpu mesa -noaudio -no-boot-anim -camera-back none -no-snapshot -no-snapshot-save -no-snapshot-load disable-animations: false - script: > - curl -L https://maven.google.com/androidx/test/services/test-services/1.6.0/test-services-1.6.0.apk --output test-services-1.6.0.apk; - adb install -r -g test-services-1.6.0.apk; - adb uninstall org.jmonkeyengine.screenshottests.android || true; - ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-android:connectedDebugAndroidTest; - exit_code=$?; - mkdir -p logcat; - adb logcat -d > logcat/logcat_full.txt || true; - adb logcat -d | grep org.jmonkeyengine.screenshottests.android > logcat/logcat.txt || true; - mkdir -p report; - adb pull /storage/emulated/0/googletest/test_outputfiles/report report/protoReport || true; - adb pull /storage/emulated/0/googletest/test_outputfiles/changed-images report/changed-images || true; - ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-proto-report:upgradeProtoReport --args="$(pwd)/report/protoReport $(pwd)/report/extentReport" || true; - echo "GRADLE_EXIT_CODE=$exit_code" >> $GITHUB_ENV; + script: | + curl -L https://maven.google.com/androidx/test/services/test-services/1.6.0/test-services-1.6.0.apk --output test-services-1.6.0.apk + adb install -r -g test-services-1.6.0.apk + exit_code=1 + for attempt in 1 2 3; do + echo "Android screenshot test attempt $attempt of 3" + adb uninstall org.jmonkeyengine.screenshottests.android || true + ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-android:connectedDebugAndroidTest + exit_code=$? + if [ $exit_code -eq 0 ]; then + break + fi + if [ $attempt -lt 3 ]; then + echo "Attempt $attempt failed, retrying..." + fi + done + mkdir -p logcat + adb logcat -d > logcat/logcat_full.txt || true + adb logcat -d | grep org.jmonkeyengine.screenshottests.android > logcat/logcat.txt || true + mkdir -p report + adb pull /storage/emulated/0/googletest/test_outputfiles/report report/protoReport || true + adb pull /storage/emulated/0/googletest/test_outputfiles/changed-images report/changed-images || true + ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-proto-report:upgradeProtoReport --args="$(pwd)/report/protoReport $(pwd)/report/extentReport" || true + echo "GRADLE_EXIT_CODE=$exit_code" >> $GITHUB_ENV exit $exit_code - name: Upload logcat uses: actions/upload-artifact@v4 From 3f77a1eace77c20b5306c33c313e0b3d0887900f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 9 Jun 2026 22:25:48 +0000 Subject: [PATCH 6/6] Fix Android retry: write script to file to avoid per-line sh -c execution --- .github/workflows/android-screenshot.yml | 56 +++++++++++++----------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/.github/workflows/android-screenshot.yml b/.github/workflows/android-screenshot.yml index c39fd0faf4..5b54300aa1 100644 --- a/.github/workflows/android-screenshot.yml +++ b/.github/workflows/android-screenshot.yml @@ -41,6 +41,36 @@ jobs: ~/.android/avd/* ~/.android/adb* key: avd-35 + - name: Prepare Android test script + run: | + cat > android-test-script.sh << 'EOF' + #!/usr/bin/env bash + curl -L https://maven.google.com/androidx/test/services/test-services/1.6.0/test-services-1.6.0.apk --output test-services-1.6.0.apk + adb install -r -g test-services-1.6.0.apk + exit_code=1 + for attempt in 1 2 3; do + echo "Android screenshot test attempt $attempt of 3" + adb uninstall org.jmonkeyengine.screenshottests.android || true + ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-android:connectedDebugAndroidTest + exit_code=$? + if [ $exit_code -eq 0 ]; then + break + fi + if [ $attempt -lt 3 ]; then + echo "Attempt $attempt failed, retrying..." + fi + done + mkdir -p logcat + adb logcat -d > logcat/logcat_full.txt || true + adb logcat -d | grep org.jmonkeyengine.screenshottests.android > logcat/logcat.txt || true + mkdir -p report + adb pull /storage/emulated/0/googletest/test_outputfiles/report report/protoReport || true + adb pull /storage/emulated/0/googletest/test_outputfiles/changed-images report/changed-images || true + ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-proto-report:upgradeProtoReport --args="$(pwd)/report/protoReport $(pwd)/report/extentReport" || true + echo "GRADLE_EXIT_CODE=$exit_code" >> $GITHUB_ENV + exit $exit_code + EOF + chmod +x android-test-script.sh - name: Run Android Screenshot Test uses: reactivecircus/android-emulator-runner@v2 with: @@ -54,31 +84,7 @@ jobs: profile: pixel_tablet emulator-options: -no-window -gpu mesa -noaudio -no-boot-anim -camera-back none -no-snapshot -no-snapshot-save -no-snapshot-load disable-animations: false - script: | - curl -L https://maven.google.com/androidx/test/services/test-services/1.6.0/test-services-1.6.0.apk --output test-services-1.6.0.apk - adb install -r -g test-services-1.6.0.apk - exit_code=1 - for attempt in 1 2 3; do - echo "Android screenshot test attempt $attempt of 3" - adb uninstall org.jmonkeyengine.screenshottests.android || true - ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-android:connectedDebugAndroidTest - exit_code=$? - if [ $exit_code -eq 0 ]; then - break - fi - if [ $attempt -lt 3 ]; then - echo "Attempt $attempt failed, retrying..." - fi - done - mkdir -p logcat - adb logcat -d > logcat/logcat_full.txt || true - adb logcat -d | grep org.jmonkeyengine.screenshottests.android > logcat/logcat.txt || true - mkdir -p report - adb pull /storage/emulated/0/googletest/test_outputfiles/report report/protoReport || true - adb pull /storage/emulated/0/googletest/test_outputfiles/changed-images report/changed-images || true - ./gradlew :jme3-screenshot-tests:jme3-screenshot-tests-proto-report:upgradeProtoReport --args="$(pwd)/report/protoReport $(pwd)/report/extentReport" || true - echo "GRADLE_EXIT_CODE=$exit_code" >> $GITHUB_ENV - exit $exit_code + script: bash android-test-script.sh - name: Upload logcat uses: actions/upload-artifact@v4 if: always()