diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml similarity index 60% rename from .github/actions/clp-execution-image-build/action.yaml rename to .github/actions/clp-build-runtime-image/action.yaml index e42c17e7fd..ecf3153994 100644 --- a/.github/actions/clp-execution-image-build/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -1,8 +1,10 @@ -name: "clp-execution-image-build" -description: "Builds a container image that contains the dependencies necessary -to run the CLP package." +name: "clp-build-runtime-image" +description: "Builds a container image to be used for running CLP." inputs: + image_type: + description: "Type of image to build" + required: true image_registry: default: "ghcr.io" description: "Container image registry" @@ -17,11 +19,11 @@ inputs: required: false platform_id: description: "Platform ID of the container (e.g. ubuntu)" - required: true + required: false platform_version_id: description: "Platform VERSION_ID / VERSION_CODENAME of the container (e.g. jammy, focal, etc.)" - required: true + required: false runs: using: "composite" @@ -45,20 +47,37 @@ runs: echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ >> "$GITHUB_OUTPUT" - - name: "Update Metadata" - id: "meta" + - name: "Compute metadata" + id: "compute-meta" + shell: "bash" + run: | + base_path="./tools/docker-images" + + if [[ "${{inputs.image_type}}" == "execution" ]]; then + platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}" + dockerfile_path="$base_path/clp-execution-base-$platform/Dockerfile" + image_name="clp-${{inputs.image_type}}-$platform" + else + dockerfile_path="$base_path/clp-package/Dockerfile" + image_name="clp-${{inputs.image_type}}" + fi + + echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT" + echo "IMAGE_NAME=$image_name" >> "$GITHUB_OUTPUT" + + - name: "Extract GitHub Metadata" + id: "extract-gh-meta" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ - /clp-execution-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" + /${{steps.compute-meta.outputs.IMAGE_NAME}}" - name: "Build and Push" if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./" - file: "./tools/docker-images/\ - clp-execution-base-${{inputs.platform_id}}-${{inputs.platform_version_id}}/Dockerfile" + file: "${{steps.compute-meta.outputs.DOCKERFILE}}" push: true - tags: "${{steps.meta.outputs.tags}}" - labels: "${{steps.meta.outputs.labels}}" + tags: "${{steps.extract-gh-meta.outputs.tags}}" + labels: "${{steps.extract-gh-meta.outputs.labels}}" diff --git a/.github/actions/run-on-image/action.yaml b/.github/actions/run-on-image/action.yaml index 25df6207a1..4e3203d26b 100644 --- a/.github/actions/run-on-image/action.yaml +++ b/.github/actions/run-on-image/action.yaml @@ -42,9 +42,11 @@ runs: - run: "./tools/scripts/deps-download/init.sh" shell: "bash" + # `--env npm_config_cache` overrides the default `/.npm` to avoid permission issues. - run: >- docker run --user $(id -u):$(id -g) + --env npm_config_cache=/tmp/.npm --volume "$GITHUB_WORKSPACE":/mnt/repo --workdir /mnt/repo ${{steps.get_image_props.outputs.qualified_image_name}} diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-artifact-build.yaml similarity index 91% rename from .github/workflows/clp-core-build.yaml rename to .github/workflows/clp-artifact-build.yaml index 4f29cec4f3..fac645c8fa 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -1,28 +1,14 @@ -name: "clp-core-build" +name: "clp-artifact-build" on: pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" + paths-ignore: &ignored_paths + - ".github/*" + - ".github/ISSUE_TEMPLATE/**" + - "components/core/tools/scripts/lib_install/macos/**" + - "docs/**" push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" + paths-ignore: *ignored_paths schedule: # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - cron: "15 0 * * *" @@ -74,31 +60,31 @@ jobs: filters: | centos_stream_9_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - "components/core/tools/scripts/lib_install/centos-stream-9/**" manylinux_2_28_x86_64_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/**" - "components/core/tools/scripts/lib_install/manylinux_2_28/**" musllinux_1_2_x86_64_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/**" - "components/core/tools/scripts/lib_install/musllinux_1_2/**" ubuntu_jammy_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" clp: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - ".gitmodules" - "components/core/cmake/**" - "components/core/CMakeLists.txt" @@ -520,3 +506,42 @@ jobs: .task/checksum/utils-cpp-lint-clang-tidy-* build/lint-clang-tidy key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" + + package-image: + name: "package-image" + if: >- + !cancelled() && !failure() && ( + needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || + needs.ubuntu-jammy-deps-image.result == 'success' + ) + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - name: "Build the package" + uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package + + - uses: "./.github/actions/clp-build-runtime-image" + with: + image_type: "package" + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index c2af595c0f..9586bbbeb5 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -3,12 +3,12 @@ name: "clp-execution-image-build" on: pull_request: paths: - - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/actions/clp-build-runtime-image/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" push: paths: - - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/actions/clp-build-runtime-image/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" schedule: @@ -61,8 +61,9 @@ jobs: shell: "bash" run: "chown $(id -u):$(id -g) -R ." - - uses: "./.github/actions/clp-execution-image-build" + - uses: "./.github/actions/clp-build-runtime-image" with: + image_type: "execution" image_registry: "ghcr.io" image_registry_username: "${{github.actor}}" image_registry_password: "${{secrets.GITHUB_TOKEN}}" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 45889bd175..f8be86614f 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -23,8 +23,10 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ openjdk-11-jdk \ pkg-config \ python3 \ + python3-dev \ python3-pip \ python3-venv \ + rsync \ software-properties-common \ unzip diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index db67f96a31..b212b5d3c1 100644 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -68,5 +68,15 @@ To clean up all build artifacts, run: task clean ``` +## Building a Docker image + +To build a Docker image containing the CLP package, run: + +```shell +task docker-images:package +``` + +This will create a Docker image named `clp-package:dev`. + [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [Task]: https://taskfile.dev/ diff --git a/docs/src/dev-docs/contributing-linting.md b/docs/src/dev-docs/contributing-linting.md index 3492e33f93..0a646bc681 100644 --- a/docs/src/dev-docs/contributing-linting.md +++ b/docs/src/dev-docs/contributing-linting.md @@ -2,7 +2,7 @@ Before submitting a PR, ensure you've run our linting tools and either fixed any violations or suppressed the warning. If you can't run the linting workflows locally, you can enable and run the -[clp-lint] and [clp-core-build] workflows in your fork. +[clp-lint] and [clp-artifact-build] workflows in your fork. ## Requirements @@ -32,7 +32,7 @@ To also apply any automatic fixes: task lint:fix ``` -[clp-core-build]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-core-build.yaml +[clp-artifact-build]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-artifact-build.yaml [clp-lint]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-lint.yaml [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&projects=&template=feature-request.yml diff --git a/docs/src/dev-docs/tooling-containers.md b/docs/src/dev-docs/tooling-containers.md index 36ee7664aa..aa7b7740cc 100644 --- a/docs/src/dev-docs/tooling-containers.md +++ b/docs/src/dev-docs/tooling-containers.md @@ -147,6 +147,16 @@ environment. tools/docker-images/clp-execution-base-ubuntu-jammy ``` +## clp-package + +An image containing the CLP package. + +* Path: + + ```text + tools/docker-images/clp-package + ``` + [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9 [core-deps-manylinux_2_28-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-manylinux_2_28 [core-deps-musllinux_1_2-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-musllinux_1_2 diff --git a/docs/src/dev-docs/tooling-gh-workflows.md b/docs/src/dev-docs/tooling-gh-workflows.md index 7ac57344c0..418a3f5648 100644 --- a/docs/src/dev-docs/tooling-gh-workflows.md +++ b/docs/src/dev-docs/tooling-gh-workflows.md @@ -3,12 +3,13 @@ The CLP repo includes several GitHub workflows for automating container image builds, artifact builds, testing, and linting. We briefly describe each workflow below. -## clp-core-build +## clp-artifact-build This workflow is responsible for: -1. building (Linux) container images containing CLP-core's dependencies, and -2. building CLP-core and running its unit tests. +1. building (Linux) container images containing CLP-core's dependencies, +2. building CLP-core and running its unit tests, and +3. building a container image containing CLP's package components. To minimize build times, the jobs in the workflow are organized in the directed acyclic graph (DAG) shown below. @@ -40,6 +41,7 @@ flowchart LR manylinux_2_28-x86_64-deps-image --> manylinux_2_28-x86_64-binaries musllinux_1_2-x86_64-deps-image --> musllinux_1_2-x86_64-binaries ubuntu-jammy-deps-image --> ubuntu-jammy-binaries + ubuntu-jammy-deps-image --> package-image ubuntu-jammy-binaries --> ubuntu-jammy-binaries-image ::: @@ -61,6 +63,7 @@ Arrows between jobs indicate a dependency. The jobs are as follows: container and runs core's unit tests. * `musllinux_1_2-x86_64-binaries`: Builds the CLP-core binaries in the built musllinux_1_2 container and runs core's unit tests. +* `package-image`: Builds a container image containing CLP's package components. * `ubuntu-jammy-binaries`: Builds the CLP-core binaries in the built Ubuntu Jammy container and runs core's unit tests. * `ubuntu-jammy-binaries-image`: Builds an Ubuntu Jammy container image containing CLP-core's diff --git a/taskfile.yaml b/taskfile.yaml index 5fee8ce8b1..874af69cf0 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -5,6 +5,7 @@ shopt: ["globstar"] includes: deps: "taskfiles/deps/main.yaml" + docker-images: "taskfiles/docker-images.yaml" docs: "taskfiles/docs.yaml" lint: "taskfiles/lint.yaml" utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" @@ -37,6 +38,7 @@ vars: {{default "" (env "CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK")}} # Checksum files + G_PACKAGE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/package.md5" G_WEBUI_CLIENT_NODE_MODULES_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui-client-node-modules.md5" G_WEBUI_COMMON_NODE_MODULES_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui-common-node-modules.md5" G_WEBUI_LOG_VIEWER_NODE_MODULES_CHECKSUM_FILE: @@ -94,7 +96,7 @@ tasks: env: NODE_ENV: "production" vars: - CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" + CHECKSUM_FILE: "{{.G_PACKAGE_CHECKSUM_FILE}}" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" sources: - "{{.G_BUILD_DIR}}/package-venv.md5" diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml new file mode 100644 index 0000000000..9a546f713c --- /dev/null +++ b/taskfiles/docker-images.yaml @@ -0,0 +1,14 @@ +version: "3" + +tasks: + package: + vars: + SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" + dir: "{{.SRC_DIR}}" + sources: + - "{{.G_PACKAGE_CHECKSUM_FILE}}" + - "{{.SRC_DIR}}/**/*" + deps: + - ":package" + cmds: + - "./build.sh" diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh index 3e71d6b4d0..9ced27320f 100755 --- a/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh +++ b/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh @@ -9,11 +9,6 @@ set -u apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ - checkinstall \ - curl \ libcurl4 \ - libmariadb-dev \ - libssl-dev \ - python3 \ - rsync \ - zstd + libmariadb3 \ + python3 diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile new file mode 100644 index 0000000000..9d1bc5c71e --- /dev/null +++ b/tools/docker-images/clp-package/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:jammy AS base + +WORKDIR /root + +COPY ./tools/docker-images/clp-package/setup-scripts ./setup-scripts +RUN ./setup-scripts/install-prebuilt-packages.sh \ + && rm -rf ./setup-scripts/ + +# Remove cached files +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +ENV CLP_HOME="/opt/clp" +ENV PATH="${CLP_HOME}/bin:${PATH}" +ENV PATH="${CLP_HOME}/sbin:${PATH}" +ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" + +USER 1000:1000 + +COPY ./build/clp-package /opt/clp + +# Flatten the image +FROM scratch +COPY --from=base / / diff --git a/tools/docker-images/clp-package/build.sh b/tools/docker-images/clp-package/build.sh new file mode 100755 index 0000000000..39df1a1105 --- /dev/null +++ b/tools/docker-images/clp-package/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +repo_root=${script_dir}/../../../ + +build_cmd=( + docker build + --tag "clp-package:dev" + "$repo_root" + --file "${script_dir}/Dockerfile" +) + +if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; +then + build_cmd+=( + --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)" + --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)" + ) +fi + +"${build_cmd[@]}" diff --git a/tools/docker-images/clp-package/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-package/setup-scripts/install-prebuilt-packages.sh new file mode 100755 index 0000000000..36e1f38793 --- /dev/null +++ b/tools/docker-images/clp-package/setup-scripts/install-prebuilt-packages.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + ca-certificates \ + libcurl4 \ + libmariadb3 \ + python3