From a338ea51242586204975652fc7ea1c46bdebd184 Mon Sep 17 00:00:00 2001 From: Gopal Lal Date: Wed, 8 Apr 2026 20:18:03 +0530 Subject: [PATCH] Fix cache warmer: run on both OS, timestamp-based cache keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Add Windows runner back to warmer matrix. GitHub Actions cache is OS-scoped — Linux caches can't be restored on Windows. 2. Run exact same Maven commands as PR CI workflows to ensure all plugins are cached. 3. Use timestamp in cache key (maven-deps-{timestamp}-{hash}) so each warmer run creates a new entry without needing manual deletion. GitHub caches are immutable. The restore step uses prefix 'maven-deps-' to match the latest entry. Old entries auto-expire after 7 days. Signed-off-by: Gopal Lal Co-authored-by: Isaac Signed-off-by: Gopal Lal --- .github/workflows/warmMavenCache.yml | 69 +++++++++++++++++----------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/.github/workflows/warmMavenCache.yml b/.github/workflows/warmMavenCache.yml index 3e88716ad..24da0f177 100644 --- a/.github/workflows/warmMavenCache.yml +++ b/.github/workflows/warmMavenCache.yml @@ -29,13 +29,15 @@ permissions: jobs: warm-cache: - # Run on Linux only. Maven repository contents (JARs/POMs) are platform-independent. - # Windows forked PRs restore this same cache via the restore-keys prefix match. - # Note: Windows runners in databricks-protected-runner-group lack bash, which - # is required for the OIDC token exchange scripts. + # Run on both Linux and Windows. GitHub Actions cache is OS-scoped — + # a cache saved on Linux cannot be restored on Windows and vice versa. + strategy: + fail-fast: false + matrix: + github-runner: [linux-ubuntu-latest, windows-server-latest] runs-on: group: databricks-protected-runner-group - labels: linux-ubuntu-latest + labels: ${{ matrix.github-runner }} steps: - name: Set up JDK @@ -131,33 +133,36 @@ jobs: run: | set -euo pipefail - # Step 1: Install all modules — resolves external dependencies from JFrog and - # installs inter-module SNAPSHOTs (e.g., jdbc-core used by assembly-thin/uber). - echo "=== Step 1: Installing all modules ===" - mvn -B install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Ddependency-check.skip=true + # Run the EXACT same Maven commands as the PR CI workflows. + # This is the only reliable way to ensure every plugin, provider, + # and transitive dependency is resolved and cached. Each command + # mirrors a real CI step from prCheck.yml, prIntegrationTests.yml, + # or coverageReport.yml. - # Step 2: Run the exact same commands that PR workflows use. - # This ensures ALL plugins, providers, and metadata are resolved and cached, - # including test-time artifacts (surefire-junit-platform, jacoco agent) and - # build-time plugins (spotless, toolchains, owasp) that mvn install alone - # doesn't trigger. + echo "=== 1/8: spotless:check (formatting-check job) ===" + mvn -B --errors spotless:check || true - # Step 2: Run a real unit test to trigger full surefire provider resolution. - # surefire-junit-platform is resolved lazily at test execution time, not at - # plugin initialization. We need at least one test to actually run. - echo "=== Step 2: Running a single unit test to resolve surefire provider ===" - mvn -B -pl jdbc-core test -Dtest="DatabricksParameterMetaDataTest#testInitialization" -Ddependency-check.skip=true || true + echo "=== 2/8: install all modules (packaging-tests job) ===" + mvn -B -pl jdbc-core,assembly-uber,assembly-thin clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true -Ddependency-check.skip=true - echo "=== Step 3: Running spotless check ===" - mvn -B --errors spotless:check || true + echo "=== 3/8: Arrow Patch Tests (unit-tests job, JDK 17+) ===" + mvn -B -Pjdk21-NioNotOpen -pl jdbc-core test -Dgroups='Jvm17PlusAndArrowToNioReflectionDisabled' -Ddependency-check.skip=true || true + + echo "=== 4/8: Arrow Allocator Tests (unit-tests job, JDK 17+) ===" + mvn -B -Pjdk21-NioNotOpen -pl jdbc-core test -Dgroups='Jvm17PlusAndArrowToNioReflectionDisabled' -Dtest="ArrowBufferAllocatorNettyManagerTest,ArrowBufferAllocatorUnsafeManagerTest,ArrowBufferAllocatorUnknownManagerTest" -DforkCount=1 -DreuseForks=false -Ddependency-check.skip=true || true + + echo "=== 5/8: Arrow Memory Tests (unit-tests job) ===" + mvn -B -Plow-memory -pl jdbc-core test -Dtest='DatabricksArrowPatchMemoryUsageTest' -Ddependency-check.skip=true || true - # Step 4: Run jacoco with a real test to resolve jacoco agent + report plugins - echo "=== Step 4: Running jacoco coverage ===" - mvn -B -pl jdbc-core test -Dtest="DatabricksParameterMetaDataTest#testInitialization" jacoco:report -Ddependency-check.skip=true || true + echo "=== 6/8: Unit Tests with jacoco (unit-tests job) ===" + mvn -B -pl jdbc-core clean test -Dtest="DatabricksParameterMetaDataTest#testInitialization" -Dgroups='!Jvm17PlusAndArrowToNioReflectionDisabled' jacoco:report -Ddependency-check.skip=true || true - echo "=== Step 5: Running integration test compilation ===" + echo "=== 7/8: Integration test compile (prIntegrationTests job) ===" mvn -B -pl jdbc-core compile test-compile -Ddependency-check.skip=true || true + echo "=== 8/8: Resolve all declared plugins ===" + mvn -B -pl jdbc-core dependency:resolve-plugins -Ddependency-check.skip=true || true + echo "Dependency resolution complete" - name: Normalize _remote.repositories before saving cache @@ -173,8 +178,20 @@ jobs: find ~/.m2/repository -name '_remote.repositories' -exec sed -i 's/jfrog-central/central/g' {} \; echo "Normalized ${COUNT} _remote.repositories markers (jfrog-central -> central)" + - name: Generate cache key with timestamp + id: cache-key + shell: bash + run: | + # Include timestamp so each warmer run creates a new cache entry + # (GitHub Actions caches are immutable — can't overwrite existing keys). + # The restore step uses prefix 'maven-deps-' to match the latest entry. + # Old entries auto-expire after 7 days of no access. + TIMESTAMP=$(date -u +%Y%m%d%H%M%S) + POM_HASH=${{ hashFiles('**/pom.xml') }} + echo "key=maven-deps-${TIMESTAMP}-${POM_HASH}" >> $GITHUB_OUTPUT + - name: Save Maven dependency cache uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: ~/.m2/repository - key: maven-deps-${{ hashFiles('**/pom.xml') }} + key: ${{ steps.cache-key.outputs.key }}