Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .github/renovate.json5
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,17 @@
"matchStrings": [
"# renovate:\\s*datasource=(?<datasource>\\S+)\\s+depName=(?<depName>\\S+)(?:\\s+versioning=(?<versioning>\\S+))?\\s+(?<currentValue>\\S+)"
]
},
{
// Update github runners used to publish images, which are maintained in packaging/images/config.json.
"customType": "regex",
"managerFilePatterns": ["/^packaging/images/config\\.json$/"],
"matchStrings": [
"\"runner\"\\s*:\\s*\"(?<depName>[a-z]+)-(?<currentValue>[0-9][0-9a-z.-]*)\""
],
"datasourceTemplate": "github-runners",
// Docker versioning preserves compatibility suffixes such as "-arm".
"versioningTemplate": "docker"
}
],
"packageRules": [
Expand Down
179 changes: 179 additions & 0 deletions .github/workflows/images.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
# Copyright The OpenTelemetry Authors
# SPDX-License-Identifier: Apache-2.0

name: Package images

on:
push:
branches: [main]
pull_request:
workflow_dispatch:

permissions:
contents: read

concurrency:
group: package-images-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
# The Changed method in packaging/images/config.go selects packages to build and publish.
# It selects the affected package for component-specific changes, or all packages for shared tooling changes.
plan:
runs-on: ubuntu-24.04
outputs:
matrix: ${{ steps.plan.outputs.matrix }}
packages: ${{ steps.plan.outputs.packages }}
platforms: ${{ steps.plan.outputs.platforms }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false

- name: Generate build plan from image configuration
id: plan
env:
EVENT: ${{ github.event_name }}
BEFORE: ${{ github.event.before }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
if [[ "$EVENT" == pull_request ]]; then
BEFORE=$(git merge-base "$BASE_SHA" HEAD)
elif [[ "$EVENT" == workflow_dispatch ]] || ! git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then
# An empty base selects all packages for manual runs or missing history.
BEFORE=""
fi
go run ./cmd/image-metadata -plan -base "$BEFORE" >> "$GITHUB_OUTPUT"

build:
needs: plan
if: needs.plan.outputs.packages != '[]'
# One native build/test job per selected package and configured architecture.
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Run Go unit tests
run: make go-unit-tests

- name: Build DEB, extract it into an image, and test package contents
env:
COMPONENT: ${{ matrix.component }}
ARCH: ${{ matrix.arch }}
GITHUB_TOKEN: ${{ github.token }}
run: make "integration-test-image-$COMPONENT" CONTAINER_ENGINE=docker

# Keep architecture subdirectories so publication can merge these artifacts.
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: image-packages-${{ matrix.component }}-${{ matrix.arch }}
path: build/images/${{ matrix.component }}/
if-no-files-found: error
retention-days: 7

publish:
needs: [plan, build]
# Only tested images from the official main branch may be published.
#
# Publication is per component: a build failure in one component must not
# withhold another component's tag, which no later run would republish --
# the next commit to affect that component carries a different version.
# Legs whose own builds produced no DEBs fail on the completeness check
# below rather than publishing a partial image.
if: >-
!cancelled() &&
needs.plan.result == 'success' &&
needs.plan.outputs.packages != '[]' &&
github.repository == 'open-telemetry/opentelemetry-packaging' &&
github.ref == 'refs/heads/main' &&
(github.event_name == 'push' || github.event_name == 'workflow_dispatch')
# Publish exactly the packages selected by plan.
strategy:
# One component's publication failure must not cancel its siblings',
# which are queued in unrelated concurrency groups.
fail-fast: false
matrix:
package: ${{ fromJSON(needs.plan.outputs.packages) }}
runs-on: ubuntu-24.04
# Retain pending publications even when native builds finish out of commit order.
concurrency:
group: package-images-publish-${{ matrix.package.component }}
queue: max
cancel-in-progress: false
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false

- name: Read package metadata
id: metadata
env:
COMPONENT: ${{ matrix.package.component }}
run: go run ./cmd/image-metadata -component "$COMPONENT" >> "$GITHUB_OUTPUT"

- name: Download tested DEBs
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: image-packages-${{ matrix.package.component }}-*
path: build/images/${{ matrix.package.component }}
merge-multiple: true

# Exactly one DEB per configured architecture. Without this a leg whose
# build failed would reach the image build with a missing or ambiguous
# context, where COPY resolves a multi-match glob to a single file.
- name: Verify the tested DEBs are complete
env:
COMPONENT: ${{ matrix.package.component }}
PACKAGE: ${{ steps.metadata.outputs.package }}
PLATFORMS: ${{ needs.plan.outputs.platforms }}
run: |
status=0
for platform in ${PLATFORMS//,/ }; do
arch="${platform#linux/}"
count=$(find "build/images/$COMPONENT/$arch" -maxdepth 1 -name "${PACKAGE}_*.deb" 2>/dev/null | wc -l | tr -d '[:space:]')
if [[ "$count" -ne 1 ]]; then
echo "::error::expected exactly one ${PACKAGE}_*.deb for $arch, found $count"
status=1
fi
done
exit "$status"

- name: Set up multi-architecture image builder
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0

- name: Log in to GHCR
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}

# Check under the publication lock, allowing different version tags to publish independently.
- name: Check for superseding inputs for this image tag
id: freshness
env:
COMPONENT: ${{ matrix.package.component }}
run: |
git fetch --no-tags origin refs/heads/main
go run ./cmd/image-metadata -component "$COMPONENT" -publish-ref FETCH_HEAD >> "$GITHUB_OUTPUT"

- name: Publish multi-architecture image
if: steps.freshness.outputs.publish == 'true'
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: build/images/${{ matrix.package.component }}
file: packaging/images/Dockerfile
platforms: ${{ needs.plan.outputs.platforms }}
push: true
tags: ghcr.io/${{ github.repository }}/${{ steps.metadata.outputs.package }}:${{ steps.metadata.outputs.version }}
build-args: PACKAGE=${{ steps.metadata.outputs.package }}
39 changes: 34 additions & 5 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
## Prerequisites

- **Go 1.26+** (package builds and tests are pure Go)
- **git** (needed by the package-image metadata unit tests, which construct git histories to exercise build-plan selection)
- **npm** (needed to fetch the Node.js auto-instrumentation agent from the npm registry)
- **Python 3 with pip** (needed to fetch the Python auto-instrumentation packages; invoked as `python3 -m pip`)
- **A container engine** (Podman or Docker — needed for local repository generation and integration tests)
- **A container engine** (Podman or Docker, needed for package-image builds, local repository generation, and integration tests).
- **`rpmbuild`** (optional — only for building the source RPM natively; `make srpm-container` runs it in a container instead)

No Ruby, FPM, or special Docker images are required to build packages.
Expand All @@ -16,6 +17,7 @@ No Ruby, FPM, or special Docker images are required to build packages.
.copr/Makefile COPR "make srpm" entry point (installs the tooling, then calls make srpm)
cmd/build-packages/ CLI entry point for building .deb and .rpm packages
cmd/build-spec/rpm/ CLI entry point for generating and staging the source RPM spec
cmd/image-metadata/ Package-image metadata and build plans
cmd/otel-config-check/ Declarative-config validator shipped inside the Python package
packaging/
builder/ Go library that drives nfpm to create packages
Expand All @@ -33,9 +35,13 @@ packaging/
dotnet/ "
python/ Config, man page template, README, requirements.txt (version pins), sitecustomize.py (plus its unit tests)
vendor/ The pyproto exporter chain, developed here with its test suites (unpublished pure-Python packages; see its README)
images/ Package-image tooling
config.json Supported packages and architectures
Dockerfile Shared package extraction and image definition
repo/ APT and YUM repository generation scripts
tests/ Integration tests
metadata/ Host-side metadata validation (no containers needed)
images/ Package-image contents and copy tests
pyprotogrpc/ Pure-Python gRPC transport tests against the otel-sink (host-side)
{python,java,nodejs,dotnet}/ Matrix E2E tests (deb+rpm × base images) asserting via the otel-sink
(python/ also hosts the sitecustomize.py interpreter compatibility tests)
Expand Down Expand Up @@ -233,11 +239,21 @@ go run ./cmd/build-packages -version <VERSION> -arch <ARCH> -format all -output
The Python package ships the `otel-config-check` validator, which the make targets cross-compile beforehand (`make otel-config-check`, producing `build/bin/otel-config-check-<arch>`).
When invoking `build-packages` directly for the Python component, build that binary first, or point `-config-check-binary` at one.

### Building package images

Build all configured package images:

```sh
make images
```

See [the image documentation](packaging/images/README.md) for configuration, image contents, naming, and individual Make tasks.

## Testing

### Go command unit tests (fast, no containers)

Unit tests for the Go commands, currently the `otel-config-check` declarative-configuration validator that ships inside the Python package.
Unit tests cover the Go commands, the package builder, package-image configuration, and build-plan selection.

```sh
make go-unit-tests
Expand Down Expand Up @@ -289,11 +305,23 @@ These tests parse `.deb` and `.rpm` files natively in Go using [pault.ag/go/debi
make integration-test-metadata
```

### Package-image tests (containers required)

Build and test all configured package images:

```sh
make integration-test-images
```

These tests are also included in `make integration-tests`.
See [the image Make tasks](packaging/images/README.md#make-tasks) for per-component targets.

### Integration tests (containers required)

End-to-end tests that install packages in Debian/Fedora containers from a local APT/YUM repository, start instrumented applications, and verify telemetry output.
End-to-end tests install packages in Debian/Fedora containers from a local APT/YUM repository, start instrumented applications, and verify telemetry output.
The aggregate target also builds and tests package images.

Run all integration tests (builds packages and local repositories first):
Run all integration tests (builds packages, local repositories, and package images first):

```sh
make integration-tests
Expand Down Expand Up @@ -377,4 +405,5 @@ To add a new language auto-instrumentation package:

## Releasing

Cutting a release and publishing the package repositories is a maintainer task documented separately in [RELEASING.md](RELEASING.md).
System-package releases and automatic package-image publication are documented separately in [RELEASING.md](RELEASING.md).
Package images follow merged upstream version pins and do not require a system-package release.
53 changes: 48 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ SHELL = /bin/bash

# Package version. Derived from the latest git tag if available, otherwise
# falls back to a development placeholder.
VERSION ?= $(shell v=`git describe --tags --match 'v[0-9]*' --abbrev=0 2>/dev/null` && echo "$${v#v}" || echo "0.0.0-dev")
VERSION ?= $(patsubst v%,%,$(shell git describe --tags --match 'v[0-9]*' --abbrev=0 2>/dev/null || echo "0.0.0-dev"))

# Target CPU architecture (amd64 or arm64).
ARCH ?= amd64
Expand Down Expand Up @@ -59,6 +59,11 @@ COMPONENTS := injector java nodejs dotnet python meta
# Languages that have integration tests (meta is excluded).
TEST_LANGUAGES := java nodejs dotnet python

IMAGE_VERSION = $(shell go run ./cmd/image-metadata -component $* -field version)
IMAGE_PACKAGE = $(shell go run ./cmd/image-metadata -component $* -field package)
IMAGE_REF = ghcr.io/open-telemetry/opentelemetry-packaging/$(IMAGE_PACKAGE):$(IMAGE_VERSION)
IMAGE_CONTEXT = build/images/$*

# ============================================================================
# Package Build Targets (nfpm, pure Go)
# ============================================================================
Expand Down Expand Up @@ -91,6 +96,36 @@ rpm-packages: otel-config-check
packages: otel-config-check
go run ./cmd/build-packages -version $(VERSION) -arch $(ARCH) -format all -output $(OUTPUT_DIR)

# ============================================================================
# Package images
# ============================================================================

.PHONY: images
.PRECIOUS: image-packages-% image-%
images:
@set -e; components="$$(go run ./cmd/image-metadata -field components)"; \
for component in $$components; do \
$(MAKE) image-$$component; \
done

image-packages-%: otel-config-check
go run ./cmd/image-metadata -component $*
rm -rf "$(IMAGE_CONTEXT)/$(ARCH)"
go run ./cmd/build-packages -version $(IMAGE_VERSION) -arch "$(ARCH)" -format deb \
-component $* -output "$(IMAGE_CONTEXT)/$(ARCH)"

image-%: image-packages-%
$(CONTAINER_ENGINE) build --platform "linux/$(ARCH)" \
--build-arg TARGETARCH="$(ARCH)" --build-arg PACKAGE=$(IMAGE_PACKAGE) \
-f packaging/images/Dockerfile -t $(IMAGE_REF) $(IMAGE_CONTEXT)

# -count=1 because the container image is not an input Go's test cache tracks:
# the test reaches it through the container engine, so a rebuilt or retagged
# image would otherwise replay a stale PASS.
integration-test-image-%: image-%
IMAGE_COMPONENT=$* IMAGE_REF=$(IMAGE_REF) IMAGE_PACKAGES_DIR=$(IMAGE_CONTEXT)/$(ARCH) \
CONTAINER_ENGINE=$(CONTAINER_ENGINE) go test -v -count=1 -timeout 5m ./packaging/tests/images/

# ============================================================================
# Source RPM (rpmbuild path, used by COPR)
# ============================================================================
Expand Down Expand Up @@ -324,9 +359,16 @@ export TESTCONTAINERS_RYUK_DISABLED = true
integration-test-metadata: packages
go test -v -timeout 5m ./packaging/tests/metadata/

.PHONY: integration-test-images
integration-test-images:
@set -e; components="$$(go run ./cmd/image-metadata -field components)"; \
for component in $$components; do \
$(MAKE) integration-test-image-$$component; \
done

.PHONY: integration-tests
integration-tests: local-repos local-apt-repo-next local-rpm-repo-next \
local-apt-vendor-repo local-rpm-vendor-repo
local-apt-vendor-repo local-rpm-vendor-repo integration-test-images
go test -v -timeout 30m ./packaging/tests/...

.PHONY: integration-test-deb-java
Expand Down Expand Up @@ -394,11 +436,12 @@ integration-test-rpm-vendor: local-rpm-repo local-rpm-vendor-repo
# Unit Tests
# ============================================================================

# Unit tests for the Go commands (e.g. the otel-config-check validator shipped
# in the Python package).
# Unit tests for Go commands, the package builder, and package-image
# configuration. The image-metadata tests construct git histories, so git must
# be on PATH; no containers or network access are needed.
.PHONY: go-unit-tests
go-unit-tests:
go test -v ./cmd/...
go test -v ./cmd/... ./packaging/builder/ ./packaging/images/

# Unit tests for sitecustomize.py. They need the `packaging` module (a runtime
# dependency of sitecustomize.py itself); a throwaway virtualenv keeps the
Expand Down
Loading
Loading