[Kafka #20707] KAFKA-19648; Cluster metadata bootstrapping with kraft checkpoint

.asf.yaml

@@ -15,14 +15,44 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# Enable the next-gen .asf.yaml parser
+meta:
+  nextgen: true
+
 notifications:
-  commits:      commits@kafka.apache.org
-  issues:       jira@kafka.apache.org
+  commits: commits@kafka.apache.org
+  issues: jira@kafka.apache.org
   pullrequests: jira@kafka.apache.org
   jira_options: link label
 
-jenkins:
-  github_whitelist:
-    - ConcurrencyPractitioner
-    - ableegoldman
-    - cadonna
+github:
+  description: "Apache Kafka - A distributed event streaming platform"
+  homepage: https://kafka.apache.org/
+  labels:
+    - java
+    - scala
+    - kafka
+    - streaming
+
+  # This list allows you to triage pull requests and trigger workflow runs on GitHub Actions. It can have a maximum of 10 collaborators.
+  # Read more here: https://github.com/apache/infrastructure-asfyaml
+  collaborators:
+    - brandboat
+    - chirag-wadhwa5
+    - DL1231
+    - m1a2st
+    - mingyen066
+    - Rancho-7
+    - sjhajharia
+    - smjn
+    - TaiJuWu
+    - Yunyung
+  enabled_merge_buttons:
+    squash: true
+    squash_commit_message: PR_TITLE_AND_DESC
+    merge: false
+    rebase: false
+
+  # Disable legacy branch protections. We have manual rulesets which protect trunk
+  # and our release branches. See INFRA-26603
+  protected_branches: ~

.github/actions/gh-api-approve-run/action.yml

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Approve Workflow Run"
+description: "Approve a Workflow run that has been submitted by a non-committer"
+inputs:
+  gh-token:
+    description: "The GitHub token for use with the CLI"
+    required: true
+  repository:
+    description: "The GitHub repository"
+    required: true
+    default: "apache/kafka"
+  run_id:
+    description: "The Workflow Run ID"
+    required: true
+  pr_number:
+    description: "The Pull Request number"
+    required: true
+  commit_sha:
+    description: "The SHA of the commit the run is for"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Approve Workflow Run
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.gh-token }}
+        REPO: ${{ inputs.repository }}
+        RUN_ID: ${{ inputs.run_id }}
+        PR_NUMBER: ${{ inputs.pr_number }}
+        COMMIT_SHA: ${{ inputs.commit_sha }}
+      run: |
+        echo "Approving workflow run $RUN_ID for PR $PR_NUMBER at SHA $COMMIT_SHA";
+        gh api --method POST \
+              -H 'Accept: application/vnd.github+json' \
+              -H 'X-GitHub-Api-Version: 2022-11-28' \
+              /repos/$REPO/actions/runs/$RUN_ID/approve

.github/actions/gh-api-update-status/action.yml

@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Update Commit Status Check"
+description: "Update the status of a commit check using the GH CLI"
+inputs:
+  # Composite actions do not support typed parameters. Everything is treated as a string
+  # See: https://github.com/actions/runner/issues/2238
+  gh-token:
+    description: "The GitHub token for use with the CLI"
+    required: true
+  repository:
+    description: "The GitHub repository"
+    required: true
+    default: "apache/kafka"
+  commit_sha:
+    description: "The SHA of the commit we are updating"
+    required: true
+  url:
+    description: "The URL of the status check"
+    required: false
+    default: ""
+  description:
+    description: "The text to display next to the check"
+    default: ""
+    required: false
+  context:
+    description: "The name of the status check"
+    required: true
+  state:
+    description: "The state of the check. Can be one of: error, failure, pending, success"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Update Check
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.gh-token }}
+        REPO: ${{ inputs.repository }}
+        COMMIT_SHA: ${{ inputs.commit_sha }}
+        STATE: ${{ inputs.state }}
+        URL: ${{ inputs.url }}
+        DESCRIPTION: ${{ inputs.description }}
+        CONTEXT: ${{ inputs.context }}
+      run: |
+        gh api --method POST -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
+        /repos/$REPO/statuses/$COMMIT_SHA \
+        -f "state=$STATE" -f "target_url=$URL" \
+        -f "description=$DESCRIPTION" \
+        -f "context=$CONTEXT"

.github/actions/run-gradle/action.yml

@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Run Tests with Gradle"
+description: "Run specified Gradle test tasks with configuration for timeout and test catalog."
+inputs:
+  # Composite actions do not support typed parameters. Everything is treated as a string
+  # See: https://github.com/actions/runner/issues/2238
+  test-task:
+    description: "The Gradle task name to run."
+    required: true
+  test-xml-output:
+    description: "Output directory for JUnit XML results"
+    required: true
+  timeout-minutes:
+    description: "The timeout for the tests, in minutes."
+    required: true
+  test-catalog-path:
+    description: "The file path of the test catalog file."
+    required: true
+  build-scan-artifact-name:
+    description: "The name to use for archiving the build scan."
+    required: true
+  test-retries:
+    description: "The number of retries for a given test should we allow"
+    required: true
+    default: "0"
+  test-repeat:
+    description: "The number of times to repeat the integration tests"
+    required: true
+    default: "1"
+  test-verbose:
+    description: "Enable additional logging by the JUnit infrastructure"
+    required: true
+    default: "false"
+  run-new-tests:
+    description: "Run tests not present in the given test catalog"
+    required: true
+    default: "false"
+  run-flaky-tests:
+    description: "Run tests marked as flaky"
+    required: true
+    default: "false"
+
+outputs:
+  gradle-exitcode:
+    description: "The result of the Gradle test task."
+    value: ${{ steps.run-tests.outputs.exitcode }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Run JUnit Tests (${{ inputs.test-task }})
+      # Gradle flags
+      # --build-cache:  Let Gradle restore the build cache
+      # --no-scan:      Don't attempt to publish the scan yet. We want to archive it first.
+      # --continue:     Keep running even if a test fails
+      # -PcommitId      Prevent the Git SHA being written into the jar files (which breaks caching)
+      shell: bash
+      id: run-tests
+      env:
+        TIMEOUT_MINUTES: ${{ inputs.timeout-minutes}}
+        TEST_CATALOG: ${{ inputs.test-catalog-path }}
+        TEST_TASK: ${{ inputs.test-task }}
+        TEST_RETRIES: ${{ inputs.test-retries }}
+        TEST_REPEAT: ${{ inputs.test-repeat }}
+        RUN_NEW_TESTS: ${{ inputs.run-new-tests }}
+        RUN_FLAKY_TESTS: ${{ inputs.run-flaky-tests }}
+        TEST_XML_OUTPUT_DIR: ${{ inputs.test-xml-output }}
+        TEST_VERBOSE: ${{ inputs.test-verbose }}
+      # This build step is invoked by build.yml to run junit tests only,
+      # Spotbugs is being run by that workflow via the "check" task and does not need to also be run here,
+      # since that is redundant.
+      run: |
+        set +e
+        ./.github/scripts/thread-dump.sh &
+        timeout ${TIMEOUT_MINUTES}m ./gradlew --build-cache --continue --no-scan \
+        -PtestLoggingEvents=started,passed,skipped,failed \
+        -PmaxParallelForks=4 \
+        -PmaxTestRetries=$TEST_RETRIES -PmaxTestRetryFailures=10 \
+        -Pkafka.test.catalog.file=$TEST_CATALOG \
+        -Pkafka.test.run.new=$RUN_NEW_TESTS \
+        -Pkafka.test.run.flaky=$RUN_FLAKY_TESTS \
+        -Pkafka.test.xml.output.dir=$TEST_XML_OUTPUT_DIR \
+        -Pkafka.cluster.test.repeat=$TEST_REPEAT \
+        -Pkafka.test.verbose=$TEST_VERBOSE \
+        -PcommitId=xxxxxxxxxxxxxxxx \
+        -x spotbugsMain \
+        -x spotbugsTest \
+        $TEST_TASK
+        exitcode="$?"
+        echo "exitcode=$exitcode" >> $GITHUB_OUTPUT
+    - name: Archive build scan (${{ inputs.test-task }})
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.build-scan-artifact-name }}
+        path: ~/.gradle/build-scan-data
+        compression-level: 9
+        if-no-files-found: ignore

.github/actions/setup-gradle/action.yml

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Gradle Setup"
+description: "Setup Java and Gradle"
+inputs:
+  # Composite actions do not support typed parameters. Everything is treated as a string
+  # See: https://github.com/actions/runner/issues/2238
+  java-version:
+    description: "Java version to use"
+    default: "17"
+  gradle-cache-read-only:
+    description: "Should the Gradle cache be read-only?"
+    default: "true"
+  gradle-cache-write-only:
+    description: "Should the Gradle cache be write-only?"
+    default: "false"
+  develocity-access-key:
+    description: "Optional access key for uploading build scans to Develocity"
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Java
+      uses: actions/setup-java@v5
+      with:
+        distribution: temurin
+        java-version: ${{ inputs.java-version }}
+    - name: Setup Gradle
+      uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2
+      env:
+        GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED: true
+      with:
+        gradle-version: wrapper
+        develocity-access-key: ${{ inputs.develocity-access-key }}
+        develocity-token-expiry: 4
+        cache-read-only: ${{ inputs.gradle-cache-read-only }}
+        cache-write-only: ${{ inputs.gradle-cache-write-only }}
+        # Cache downloaded JDKs in addition to the default directories.
+        gradle-home-cache-includes: |
+          caches
+          notifications
+          jdks
+        cache-cleanup: on-success

.github/actions/setup-python/action.yml

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "Python Setup"
+description: "Setup Python and install dependencies"
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: 3.12
+    - name: Pip install
+      shell: bash
+      run: pip install -r .github/scripts/requirements.txt