diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a813a8a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git +.github +.gitignore +.claude +helpers +*.md diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml new file mode 100644 index 0000000..97be5da --- /dev/null +++ b/.github/workflows/docker-release.yml @@ -0,0 +1,164 @@ +name: Docker Release & Latest + +on: + release: + types: [published] + push: + branches: + - main + paths: + - "**" + - "!.github/**" + - "!**.md" + - "!docker-compose.yml" + workflow_dispatch: + inputs: + build_type: + description: "Build type" + required: true + default: "latest" + type: choice + options: + - latest + - release + - both + +permissions: + contents: read + packages: write + +jobs: + build_and_push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: "amd64,arm64" + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to Ghcr + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Determine build type and version + id: version + run: | + if [[ "${{ github.event_name }}" == "release" ]]; then + VERSION="${{ github.event.release.tag_name }}" + BUILD_TYPE="release" + elif [[ "${{ github.event_name }}" == "push" ]]; then + VERSION="latest" + BUILD_TYPE="latest" + else + BUILD_TYPE="${{ github.event.inputs.build_type }}" + VERSION="latest" + fi + + echo "build_type=${BUILD_TYPE}" >> $GITHUB_OUTPUT + echo "version=${VERSION}" >> $GITHUB_OUTPUT + + STRIPPED_VERSION=$(echo "${VERSION}" | sed 's/^v//') + echo "stripped_version=${STRIPPED_VERSION}" >> $GITHUB_OUTPUT + echo "Version: ${VERSION}, Stripped: ${STRIPPED_VERSION}, Type: ${BUILD_TYPE}" + + # === LATEST BUILD (push to main or workflow_dispatch) === + + - name: Build and push latest (primary) + if: steps.version.outputs.build_type == 'latest' || steps.version.outputs.build_type == 'both' + uses: docker/build-push-action@v5 + with: + context: ./ + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: | + ghcr.io/shuffle/orborus:latest + ghcr.io/shuffle/orborus:2.2.1 + + - name: Copy latest to DockerHub + if: steps.version.outputs.build_type == 'latest' || steps.version.outputs.build_type == 'both' + continue-on-error: true + run: | + docker buildx imagetools create \ + --tag ${{ secrets.DOCKERHUB_USERNAME }}/orborus:latest \ + --tag ${{ secrets.DOCKERHUB_USERNAME }}/orborus:2.2.1 \ + ghcr.io/shuffle/orborus:latest + + - name: Copy latest to shuffle DockerHub + if: steps.version.outputs.build_type == 'latest' || steps.version.outputs.build_type == 'both' + continue-on-error: true + run: | + docker buildx imagetools create \ + --tag shuffle/orborus:latest \ + --tag shuffle/orborus:2.2.1 \ + ghcr.io/shuffle/orborus:latest + + - name: Copy latest to frikky + if: steps.version.outputs.build_type == 'latest' || steps.version.outputs.build_type == 'both' + continue-on-error: true + run: | + docker buildx imagetools create \ + --tag frikky/orborus:latest \ + --tag frikky/orborus:2.2.1 \ + --tag frikky/shuffle:orborus \ + ghcr.io/shuffle/orborus:latest + + # === RELEASE BUILD (GitHub release or workflow_dispatch) === + + - name: Build and push release (primary) + if: steps.version.outputs.build_type == 'release' || steps.version.outputs.build_type == 'both' + uses: docker/build-push-action@v5 + with: + context: ./ + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: | + ghcr.io/shuffle/orborus:${{ steps.version.outputs.version }} + ghcr.io/shuffle/orborus:${{ steps.version.outputs.stripped_version }} + + - name: Copy release to DockerHub + if: steps.version.outputs.build_type == 'release' || steps.version.outputs.build_type == 'both' + continue-on-error: true + run: | + docker buildx imagetools create \ + --tag ${{ secrets.DOCKERHUB_USERNAME }}/orborus:${{ steps.version.outputs.version }} \ + --tag ${{ secrets.DOCKERHUB_USERNAME }}/orborus:${{ steps.version.outputs.stripped_version }} \ + ghcr.io/shuffle/orborus:${{ steps.version.outputs.version }} + + - name: Copy release to shuffle DockerHub + if: steps.version.outputs.build_type == 'release' || steps.version.outputs.build_type == 'both' + continue-on-error: true + run: | + docker buildx imagetools create \ + --tag shuffle/orborus:${{ steps.version.outputs.version }} \ + --tag shuffle/orborus:${{ steps.version.outputs.stripped_version }} \ + ghcr.io/shuffle/orborus:${{ steps.version.outputs.version }} + + - name: Copy release to frikky + if: steps.version.outputs.build_type == 'release' || steps.version.outputs.build_type == 'both' + continue-on-error: true + run: | + docker buildx imagetools create \ + --tag frikky/orborus:${{ steps.version.outputs.version }} \ + --tag frikky/orborus:${{ steps.version.outputs.stripped_version }} \ + ghcr.io/shuffle/orborus:${{ steps.version.outputs.version }} diff --git a/.github/workflows/dockerbuild.yml b/.github/workflows/dockerbuild.yml deleted file mode 100644 index 492c040..0000000 --- a/.github/workflows/dockerbuild.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: latest-dockerbuild - -on: - workflow_dispatch: - push: - branches: - - main - paths: - - "**" - - "!.github/**" - - "!**.md" - - "!docker-compose.yml" -permissions: - contents: read - packages: write - -jobs: - main: - runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} - strategy: - fail-fast: false - matrix: - include: - - app: orborus - path: ./ - version: latest - experimental: true - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: "amd64,arm64,arm" - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to Ghcr - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Ghcr Build and push - id: docker_build - uses: docker/build-push-action@v5 - with: - context: ${{ matrix.path }}/ - file: ${{ matrix.path }}/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - tags: | - ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ matrix.version }} - ${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ matrix.version }} - frikky/shuffle-${{ matrix.app }}:${{ matrix.version }} - frikky/shuffle:${{ matrix.app }} - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/release_binaries.yml b/.github/workflows/release_binaries.yml index 9975acb..09929b4 100644 --- a/.github/workflows/release_binaries.yml +++ b/.github/workflows/release_binaries.yml @@ -4,66 +4,94 @@ on: workflow_dispatch: inputs: version: - description: 'Version of the release (e.g. v0.3.6)' + description: "Version (e.g. v0.3.6)" required: true +permissions: + contents: write + jobs: build: runs-on: ubuntu-latest + strategy: matrix: os: [linux, darwin, windows] arch: [amd64, arm64] + steps: - name: Checkout source uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Go uses: actions/setup-go@v5 with: go-version: "1.24" - - name: Set version - run: echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV - - name: Build binary run: | mkdir -p dist - export GOOS=${{ matrix.os }} - export GOARCH=${{ matrix.arch }} + + GOOS=${{ matrix.os }} + GOARCH=${{ matrix.arch }} + EXT="" LDFLAGS="" - + if [ "$GOOS" = "windows" ]; then EXT=".exe" LDFLAGS="-H=windowsgui" fi - - BINARY_NAME=orborus-agent-${GOOS}-${GOARCH} - OUTPUT=dist/$BINARY_NAME$EXT - + + BINARY_NAME="orborus-agent-${GOOS}-${GOARCH}" + OUTPUT="dist/${BINARY_NAME}${EXT}" + echo "Building $OUTPUT" - GOOS=$GOOS GOARCH=$GOARCH go build -ldflags="$LDFLAGS" -o $OUTPUT + GOOS=$GOOS GOARCH=$GOARCH go build -ldflags="$LDFLAGS" -o "$OUTPUT" + - name: Package run: | cd dist - FILENAME=orborus-agent-${{ matrix.os }}-${{ matrix.arch }} - if [ "${{ matrix.os }}" = "windows" ]; then - zip $FILENAME.zip $FILENAME.exe - else - tar -czf $FILENAME.tar.gz $FILENAME - fi - - name: Upload build artifact + VERSION="${{ github.event.inputs.version }}" + + for file in *; do + if [[ "$file" == *.exe ]]; then + zip "${file%.exe}.zip" "$file" + else + tar -czf "${file}.tar.gz" "$file" + fi + done + + - name: Upload build artifacts uses: actions/upload-artifact@v4 with: name: binaries-${{ matrix.os }}-${{ matrix.arch }} path: dist/* + release: needs: build runs-on: ubuntu-latest + steps: + - name: Checkout source (for tag creation) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create and push git tag + run: | + VERSION="${{ github.event.inputs.version }}" + + git config user.name "github-actions" + git config user.email "github-actions@github.com" + + git tag "$VERSION" + git push origin "$VERSION" + - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -73,11 +101,9 @@ jobs: run: | find dist -mindepth 2 -type f -exec mv -t dist {} + || true - - name: Upload binaries to GitHub Release + - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: tag_name: ${{ github.event.inputs.version }} files: dist/* - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index d535bc5..9c80a4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM golang:1.25 AS builder +FROM --platform=$BUILDPLATFORM golang:1.25 AS builder +ARG TARGETOS TARGETARCH WORKDIR /app @@ -6,16 +7,15 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -o /app/orborus . +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o /app/orborus . FROM alpine:3.22.1 RUN apk add --no-cache bash tzdata -#COPY --from=builder /app/orborus orborus -COPY --from=builder /app/ / +COPY --from=builder /app/orborus /orborus ENV ENVIRONMENT_NAME=Shuffle \ BASE_URL=http://shuffle-backend:5001 \ - DOCKER_API_VERSION=1.40 \ + DOCKER_API_VERSION=1.44 \ SHUFFLE_OPENSEARCH_URL=https://opensearch:9200 CMD ["./orborus"] diff --git a/helpers/docker-compose.yml b/helpers/docker-compose.yml index ef6132e..e3d680a 100644 --- a/helpers/docker-compose.yml +++ b/helpers/docker-compose.yml @@ -12,7 +12,7 @@ services: - ORG_ID=Shuffle - ENVIRONMENT_NAME=Shuffle - BASE_URL=http://192.168.86.39:5001 - - DOCKER_API_VERSION=1.40 + - DOCKER_API_VERSION=1.44 - SHUFFLE_SCALE_REPLICAS=5 - SHUFFLE_SWARM_CONFIG=run restart: unless-stopped diff --git a/helpers/run.sh b/helpers/run.sh index d71ac59..a8566a9 100755 --- a/helpers/run.sh +++ b/helpers/run.sh @@ -10,7 +10,7 @@ # ghcr.io/frikky/shuffle-orborus:nightly docker run \ - --env DOCKER_API_VERSION=1.40 \ + --env DOCKER_API_VERSION=1.44 \ --env ENVIRONMENT_NAME="Another env" \ --env ORG="2e7b6a08-b63b-4fc2-bd70-718091509db1" \ --env AUTH="env auth" \ diff --git a/orborus.go b/orborus.go index c7e697e..a893ad2 100755 --- a/orborus.go +++ b/orborus.go @@ -1,7 +1,7 @@ package main /* - Orborus exists to listen for new jobs from Shuffle. This is to run workflows, pipelines, and other tasks. + Orborus exists to listen for new jobs from Shuffle. This is to run workflows, pipelines, and other tasks. */ import ( "archive/zip" @@ -697,15 +697,11 @@ func deployServiceWorkers(image string) { Replicas: &replicatedJobs, }, }, - Networks: func() []swarm.NetworkAttachmentConfig { - networks := []swarm.NetworkAttachmentConfig{ - {Target: networkID}, - } - if strings.ToLower(os.Getenv("SHUFFLE_ATTACH_INGRESS_NETWORK")) != "false" { - networks = append(networks, swarm.NetworkAttachmentConfig{Target: "ingress"}) - } - return networks - }(), + Networks: []swarm.NetworkAttachmentConfig{ + swarm.NetworkAttachmentConfig{ + Target: networkID, + }, + }, EndpointSpec: &swarm.EndpointSpec{ Mode: "vip", Ports: []swarm.PortConfig{ @@ -802,6 +798,15 @@ func deployServiceWorkers(image string) { serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))) } + if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_APP_HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_APP_HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_APP_NO_PROXY=%s", os.Getenv("NO_PROXY"))) + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_APP_no_proxy=%s", os.Getenv("no_proxy"))) + } + + serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_PASS_APP_PROXY=%s", os.Getenv("SHUFFLE_PASS_APP_PROXY"))) + if len(workerServerUrl) > 0 { serviceSpec.TaskTemplate.ContainerSpec.Env = append(serviceSpec.TaskTemplate.ContainerSpec.Env, fmt.Sprintf("SHUFFLE_WORKER_SERVER_URL=%s", os.Getenv("SHUFFLE_WORKER_SERVER_URL"))) } @@ -879,22 +884,38 @@ func deployServiceWorkers(image string) { services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{}) if serr == nil { for _, svc := range services { - if svc.Spec.Annotations.Name == innerContainerName { - log.Printf("[DEBUG] Found service %s (%s) — patching network attach", innerContainerName, svc.ID) - - spec := svc.Spec - spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{ - Target: networkID, - }) + if svc.Spec.Annotations.Name != innerContainerName { + continue + } - _, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{}) - if uerr != nil { - log.Printf("[WARNING] Failed to patch service %s with network %s: %v", innerContainerName, networkID, uerr) - } else { - log.Printf("[INFO] Successfully attached network %s to service %s", networkID, innerContainerName) + // Check if the network already exists or not + networkFound := false + for _, net := range svc.Spec.TaskTemplate.Networks { + if net.Target == networkID { + networkFound = true + break } + } + + if networkFound { + log.Printf("[DEBUG] Network %s already attached to service %s, skipping patch", networkID, innerContainerName) break } + + log.Printf("[DEBUG] Found service %s (%s) — patching network attach", innerContainerName, svc.ID) + + spec := svc.Spec + spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{ + Target: networkID, + }) + + _, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{}) + if uerr != nil { + log.Printf("[WARNING] Failed to patch service %s with network %s: %v", innerContainerName, networkID, uerr) + } else { + log.Printf("[INFO] Successfully attached network %s to service %s", networkID, innerContainerName) + } + break } } else { log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr) @@ -1862,6 +1883,11 @@ func initializeImages() { log.Printf("[DEBUG] Setting swarm config to %#v. Default is empty.", swarmConfig) + if len(workerVersion) == 0 { + workerVersion = "latest" + log.Printf("[INFO] SHUFFLE_WORKER_VERSION not defined. Defaulting to %#v", workerVersion) + } + // This is now always static newWorker := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { @@ -2552,6 +2578,11 @@ func mainLoop() { ctx := context.Background() workerTimeout := 600 + if len(workerVersion) == 0 { + workerVersion = "latest" + log.Printf("[INFO] SHUFFLE_WORKER_VERSION not defined. Defaulting to %#v", workerVersion) + } + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage @@ -2771,7 +2802,6 @@ func mainLoop() { // FIXME - during init, BUILD and/or LOAD worker and app_sdk // Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk log.Printf("[INFO] Setting up Docker environment. Downloading worker and App SDK!") - initializeImages() if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" { @@ -4359,7 +4389,8 @@ func removeFile(fileName string) error { containerName := "tenzir-node" srcPath := fmt.Sprintf("/var/lib/tenzir/sigma_rules/%s", fileName) - checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + //checkSrcCmd := exec.Command("docker", "exec", containerName, "sh", "-c", fmt.Sprintf("test -f %s", srcPath)) + checkSrcCmd := exec.Command("docker", "exec", containerName, "test", "-f", srcPath) if err := checkSrcCmd.Run(); err != nil { // If the file does not exist, simply return nil if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 { @@ -4776,6 +4807,10 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, // Never happening but okay if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { + if workerVersion == "" { + workerVersion = "latest" + } + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage @@ -4809,6 +4844,10 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string, } if strings.Contains(fmt.Sprintf("%s", err), "connection refused") || strings.Contains(fmt.Sprintf("%s", err), "EOF") { + if len(workerVersion) == 0 { + workerVersion = "latest" + } + workerImage := fmt.Sprintf("ghcr.io/shuffle/shuffle-worker:%s", workerVersion) if len(newWorkerImage) > 0 { workerImage = newWorkerImage