From dec1e4e969995813f82ffe8da80358c636b4992b Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Sat, 28 Mar 2026 22:13:58 +0100 Subject: [PATCH] Migrate antithesis to K8s Signed-off-by: Marek Siarkowicz --- .github/workflows/antithesis-test.yml | 4 +- .github/workflows/antithesis-verify.yml | 48 ++---- tests/antithesis/Makefile | 42 ++--- tests/antithesis/README.md | 143 ++++-------------- tests/antithesis/config/Dockerfile | 16 +- .../config/docker-compose-1-node.yml | 62 -------- .../config/docker-compose-3-node.yml | 124 --------------- tests/antithesis/config/manifests/client.yaml | 42 +++++ .../manifests/default-etcd-3-replicas.yaml | 16 +- 9 files changed, 114 insertions(+), 383 deletions(-) delete mode 100644 tests/antithesis/config/docker-compose-1-node.yml delete mode 100644 tests/antithesis/config/docker-compose-3-node.yml create mode 100644 tests/antithesis/config/manifests/client.yaml diff --git a/.github/workflows/antithesis-test.yml b/.github/workflows/antithesis-test.yml index 8db6a114b4d6..7da5146d62b0 100644 --- a/.github/workflows/antithesis-test.yml +++ b/.github/workflows/antithesis-test.yml @@ -78,7 +78,7 @@ jobs: - name: Run Antithesis Tests uses: antithesishq/antithesis-trigger-action@f6221e2ba819fe0ac3e36bd67a281fa439a03fba # v0.10 with: - notebook_name: etcd + notebook_name: etcd_k8s tenant: linuxfoundation username: ${{ secrets.ANTITHESIS_WEBHOOK_USERNAME }} password: ${{ secrets.ANTITHESIS_WEBHOOK_PASSWORD }} @@ -89,5 +89,5 @@ jobs: email_recipients: ${{ inputs.email || 'siarkowicz@google.com' }} test_name: ${{ inputs.test || 'etcd nightly antithesis run' }} additional_parameters: |- - custom.duration = ${{ inputs.duration || 12 }} + antithesis.duration = ${{ inputs.duration || 12 }} antithesis.source = ${{ inputs.etcd_ref || 'main' }} diff --git a/.github/workflows/antithesis-verify.yml b/.github/workflows/antithesis-verify.yml index 1341935cdf3e..1b8e0fcbda90 100644 --- a/.github/workflows/antithesis-verify.yml +++ b/.github/workflows/antithesis-verify.yml @@ -1,5 +1,5 @@ --- -name: Verify Antithesis Docker Compose Pipeline +name: Antithesis Verify permissions: contents: read @@ -12,61 +12,35 @@ on: - 'tests/antithesis/**' - '.github/workflows/antithesis-verify.yml' pull_request: + branches: + - main paths: - 'tests/antithesis/**' - '.github/workflows/antithesis-verify.yml' jobs: - test-docker-compose: - strategy: - matrix: - node-count: [1, 3] - name: Test ${{ matrix.node-count }}-node cluster + test: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Build etcd-server and etcd-client images + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Build images run: | make -C tests/antithesis antithesis-build-etcd-image make -C tests/antithesis antithesis-build-client-docker-image - - name: Run docker-compose up + - name: Set up Kubernetes cluster working-directory: ./tests/antithesis - run: | - make antithesis-docker-compose-up CFG_NODE_COUNT=${{ matrix.node-count }} & - - - name: Check for healthy cluster - working-directory: ./tests/antithesis - run: | - timeout=120 - interval=10 - end_time=$(( $(date +%s) + timeout )) - - while [ $(date +%s) -lt $end_time ]; do - # The client container might not be running yet, so ignore errors from docker compose logs - if docker compose -f config/docker-compose-${{ matrix.node-count }}-node.yml logs client 2>/dev/null | grep -q "Client \[entrypoint\]: cluster is healthy!"; then - echo "Cluster is healthy!" - exit 0 - fi - echo "Waiting for cluster to become healthy..." - sleep $interval - done - - echo "Cluster did not become healthy in ${timeout} seconds." - docker compose -f config/docker-compose-${{ matrix.node-count }}-node.yml logs - exit 1 + run: make antithesis-k8s-up - name: Run traffic working-directory: ./tests/antithesis - run: make antithesis-run-container-traffic CFG_NODE_COUNT=${{ matrix.node-count }} + run: make antithesis-run-k8s-traffic - name: Run validation working-directory: ./tests/antithesis - run: make antithesis-run-container-validation CFG_NODE_COUNT=${{ matrix.node-count }} + run: make antithesis-run-k8s-validation - name: Clean up if: always() working-directory: ./tests/antithesis - run: make antithesis-clean CFG_NODE_COUNT=${{ matrix.node-count }} + run: make antithesis-clean diff --git a/tests/antithesis/Makefile b/tests/antithesis/Makefile index c2a00b56e0d1..993562d3e3a6 100644 --- a/tests/antithesis/Makefile +++ b/tests/antithesis/Makefile @@ -45,41 +45,41 @@ antithesis-build-etcd-image-main: REF=main antithesis-build-etcd-image-main: antithesis-build-etcd-image .PHONY: antithesis-build-config-image -antithesis-build-config-image: validate-node-count +antithesis-build-config-image: docker build -f config/Dockerfile config -t etcd-config:latest \ - --build-arg IMAGE_TAG=$(IMAGE_TAG) \ - --build-arg NODE_COUNT=$(CFG_NODE_COUNT) + --build-arg IMAGE_TAG=$(IMAGE_TAG) -.PHONY: antithesis-docker-compose-up -antithesis-docker-compose-up: validate-node-count - export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \ - docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml up +.PHONY: antithesis-k8s-up +antithesis-k8s-up: check-k8s-tools + kind create cluster --name etcd-antithesis + kind load docker-image etcd-client:latest --name etcd-antithesis + kind load docker-image etcd-server:latest --name etcd-antithesis + kubectl apply -f config/manifests + kubectl wait --for=condition=ready pod -l app=etcd --timeout=120s + kubectl wait --for=condition=ready pod -l app=etcd-client --timeout=120s -.PHONY: antithesis-run-container-traffic -antithesis-run-container-traffic: validate-node-count - export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \ - docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml exec client /opt/antithesis/test/v1/robustness/singleton_driver_traffic +.PHONY: antithesis-run-k8s-traffic +antithesis-run-k8s-traffic: + kubectl exec deployment/etcd-client -- /opt/antithesis/test/v1/robustness/singleton_driver_traffic -.PHONY: antithesis-run-container-validation -antithesis-run-container-validation: validate-node-count - export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \ - docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml exec client /opt/antithesis/test/v1/robustness/finally_validation +.PHONY: antithesis-run-k8s-validation +antithesis-run-k8s-validation: + kubectl exec deployment/etcd-client -- /opt/antithesis/test/v1/robustness/finally_validation .PHONY: antithesis-run-local-traffic antithesis-run-local-traffic: - export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata0,/tmp/etcddata1,/tmp/etcddata2 && export ETCD_ROBUSTNESS_REPORT_PATH=report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \ + export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata/etcd-0,/tmp/etcddata/etcd-1,/tmp/etcddata/etcd-2 && export ETCD_ROBUSTNESS_REPORT_PATH=reports/report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \ go run --race ./test-template/robustness/traffic/main.go .PHONY: antithesis-run-local-validation antithesis-run-local-validation: - export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata0,/tmp/etcddata1,/tmp/etcddata2 && export ETCD_ROBUSTNESS_REPORT_PATH=report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \ + export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata/etcd-0,/tmp/etcddata/etcd-1,/tmp/etcddata/etcd-2 && export ETCD_ROBUSTNESS_REPORT_PATH=reports/report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \ go run --race ./test-template/robustness/finally/main.go .PHONY: antithesis-clean -antithesis-clean: validate-node-count - export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \ - docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml down --remove-orphans - rm -rf /tmp/etcddata0 /tmp/etcddata1 /tmp/etcddata2 /tmp/etcdreport +antithesis-clean: check-k8s-tools + kind delete cluster --name etcd-antithesis + rm -rf /tmp/etcddata /tmp/etcdreport .PHONY: validate-node-count validate-node-count: diff --git a/tests/antithesis/README.md b/tests/antithesis/README.md index a7f93978def5..f8f969881671 100644 --- a/tests/antithesis/README.md +++ b/tests/antithesis/README.md @@ -17,7 +17,7 @@ For more details on robustness tests, see the [robustness directory](../robustne ## Antithesis Setup The setup consists of a 3-node etcd cluster and a client container, orchestrated -via [Docker Compose](https://antithesis.com/docs/getting_started/setup/). +via [Kubernetes](https://antithesis.com/docs/getting_started/setup_k8s/). During the etcd Antithesis test suite the etcd server is built with the following patches: @@ -52,11 +52,18 @@ in the following way: [Singleton Driver Command]: https://antithesis.com/docs/test_templates/test_composer_reference/#singleton-driver [Finally Command]: https://antithesis.com/docs/test_templates/test_composer_reference/#finally-command -# Running tests with docker compose +## Running tests locally with Kubernetes + +### Prerequisites + +Please make sure that you have the following tools installed on your local: + +* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) +* [kind](https://kind.sigs.k8s.io/docs/user/quick-start#installation) ## Quickstart -### 1. Build and Tag the Docker Image +### 1. Build and Tag the Docker Images Run this command from the `antithesis/test-template` directory: @@ -71,140 +78,42 @@ Both commands build etcd-server and etcd-client from the current branch. To buil make antithesis-build-etcd-image REF=${GIT_REF} ``` -### 2. (Optional) Check the Image Locally - -You can verify your new image is built: - -```bash -docker images | grep etcd-client -``` - -It should show something like: - -``` -etcd-client latest -``` - -### 3. Use in Docker Compose +#### 2. Set up the Kubernetes cluster Run the following command from the root directory for Antithesis tests (`tests/antithesis`): ```bash -make antithesis-docker-compose-up +make antithesis-k8s-up ``` -The command uses the etcd client and server images built from step 1. +This command will: -The client will continuously check the health of the etcd nodes and print logs similar to: +* Create a local `kind` cluster named `etcd-antithesis`. +* Load the built `etcd-client` and `etcd-server` images into the cluster. +* Deploy the Kubernetes manifests from `config/manifests`. -``` -[+] Running 4/4 - ✔ Container etcd0 Created 0.0s - ✔ Container etcd2 Created 0.0s - ✔ Container etcd1 Created 0.0s - ✔ Container client Recreated 0.1s -Attaching to client, etcd0, etcd1, etcd2 -etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.134294Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ADVERTISE_CLIENT_URLS","variable-value":"http://etcd2.etcd:2379"} -etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.138501Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_ADVERTISE_PEER_URLS","variable-value":"http://etcd2:2380"} -etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.138646Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER","variable-value":"etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380"} -etcd0 | {"level":"info","ts":"2025-04-14T07:23:25.138434Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ADVERTISE_CLIENT_URLS","variable-value":"http://etcd0.etcd:2379"} -etcd0 | {"level":"info","ts":"2025-04-14T07:23:25.138582Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_ADVERTISE_PEER_URLS","variable-value":"http://etcd0:2380"} -etcd0 | {"level":"info","ts":"2025-04-14T07:23:25.138592Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER","variable-value":"etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380"} - -... -... -(skipping some repeated logs for brevity) -... -... - -etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.484698Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} -etcd1 | {"level":"info","ts":"2025-04-14T07:23:25.484092Z","caller":"embed/serve.go:210","msg":"serving client traffic insecurely; this is strongly discouraged!","traffic":"grpc+http","address":"[::]:2379"} -etcd0 | {"level":"info","ts":"2025-04-14T07:23:25.484563Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} -etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.485101Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"} -etcd1 | {"level":"info","ts":"2025-04-14T07:23:25.484130Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"} -etcd2 | {"level":"info","ts":"2025-04-14T07:23:25.485782Z","caller":"embed/serve.go:210","msg":"serving client traffic insecurely; this is strongly discouraged!","traffic":"grpc+http","address":"[::]:2379"} -etcd1 | {"level":"info","ts":"2025-04-14T07:23:25.484198Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"} -client | Client [entrypoint]: starting... -client | Client [entrypoint]: checking cluster health... -client | Client [entrypoint]: connection successful with etcd0 -client | Client [entrypoint]: connection successful with etcd1 -client | Client [entrypoint]: connection successful with etcd2 -client | Client [entrypoint]: cluster is healthy! -``` +The client pod will continuously check the health of the etcd nodes and wait for the cluster to be healthy. -And it will stay running indefinitely. +#### 3. Running the tests -### 4. Running the tests +To run the tests locally against the deployed cluster: ```bash -make antithesis-run-container-traffic -make antithesis-run-container-validation +make antithesis-run-k8s-traffic +make antithesis-run-k8s-validation ``` -Alternatively, with the etcd cluster from step 3, to run the tests locally without rebuilding the client image: - -```bash -make antithesis-run-local-traffic -make antithesis-run-local-validation -``` +#### 4. Prepare for next run -### 5. Prepare for next run - -Unfortunatelly robustness tests don't support running on non empty database. +Unfortunately robustness tests don't support running on a non-empty database. So for now you need to cleanup the storage before repeating the run or you will get "non empty database at start, required by model used for linearizability validation" error. ```bash make antithesis-clean ``` -## Troubleshooting - -- **Image Pull Errors**: If Docker can’t pull `etcd-client:latest`, make sure you built it locally (see the “Build and Tag” step) or push it to a registry that Compose can access. - -# Running Tests with Kubernetes (WIP) - -## Prerequisites - -Please make sure that you have the following tools installed on your local: - -- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) -- [kind](https://kind.sigs.k8s.io/docs/user/quick-start#installation) - -## Testing locally - -### Setting up the cluster and deploying the images - -#### 1. Ensure your access to a test kubernetes cluster - -You can use `kind` to create a local cluster to deploy the etcd-server and test client. Once you have `kind` installed, you can use the following command to create a local cluster: - -```bash -kind create cluster -``` - -Alternatively, you can use any existing kubernetes cluster you have access to. - -#### 2. Build and load the images - -Please [build the client and server images](#1-build-and-tag-the-docker-image) first. Then load the images into the `kind` cluster: +This command will delete the `kind` cluster and remove the local data directories. -If you use `kind`, the cluster will need to have access to the images using the following commands: +### Troubleshooting -```bash -kind load docker-image etcd-client:latest -kind load docker-image etcd-server:latest -``` - -If you use something other than `kind`, please make sure the images are accessible to your cluster. This might involve pushing the images to a container registry that your cluster can pull from. - -#### 3. Deploy the kubernetes manifests - -```bash -kubectl apply -f ./config/manifests -``` - -### Tearing down the cluster - -```bash -kind delete cluster --name kind -``` +* **Image Pull Errors**: If Kubernetes can’t pull `etcd-client:latest`, make sure you built it locally and that it was successfully loaded into the `kind` cluster. diff --git a/tests/antithesis/config/Dockerfile b/tests/antithesis/config/Dockerfile index 7cd60d752fb6..efdbee5283de 100644 --- a/tests/antithesis/config/Dockerfile +++ b/tests/antithesis/config/Dockerfile @@ -1,12 +1,8 @@ -ARG GO_VERSION=1.25.5 - -FROM golang:$GO_VERSION AS build -RUN go install github.com/a8m/envsubst/cmd/envsubst@v1.4.3 - -ARG IMAGE_TAG -ARG NODE_COUNT -COPY docker-compose-${NODE_COUNT}-node.yml /docker-compose.yml.template -RUN IMAGE_TAG=${IMAGE_TAG} cat /docker-compose.yml.template | envsubst > /docker-compose.yml +FROM alpine AS build +ARG IMAGE_TAG=latest +COPY manifests/ /manifests/ +RUN sed -i "s/etcd-client:latest/etcd-client:${IMAGE_TAG}/g" /manifests/client.yaml && \ + sed -i "s/etcd-server:latest/etcd-server:${IMAGE_TAG}/g" /manifests/default-etcd-3-replicas.yaml FROM scratch -COPY --from=build /docker-compose.yml /docker-compose.yml +COPY --from=build /manifests/ /manifests/ diff --git a/tests/antithesis/config/docker-compose-1-node.yml b/tests/antithesis/config/docker-compose-1-node.yml deleted file mode 100644 index 265f5fd9e84b..000000000000 --- a/tests/antithesis/config/docker-compose-1-node.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- -services: - # This is needed for creating non-root data folders on host. - # By default, if the folders don't exist when mounting, compose creates them with root as owner. - # With root owner, accessing the WAL files from local tests will fail due to an unauthorized access error. - init: - image: 'docker.io/library/ubuntu:latest' - user: root - group_add: - - '${GROUP_ID:-root}' - volumes: - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcddata0 - - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report - command: - - /bin/sh - - -c - - | - rm -rf /var/etcddata0/* /var/report/* - chown -R ${USER_ID:-root}:${GROUP_ID:-root} /var/etcddata0 /var/report - - etcd0: - image: 'etcd-server:${IMAGE_TAG:-latest}' - container_name: etcd0 - hostname: etcd0 - environment: - ETCD_NAME: "etcd0" - ETCD_INITIAL_ADVERTISE_PEER_URLS: "http://etcd0:2380" - ETCD_LISTEN_PEER_URLS: "http://0.0.0.0:2380" - ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" - ETCD_ADVERTISE_CLIENT_URLS: "http://etcd0.etcd:2379" - ETCD_INITIAL_CLUSTER_TOKEN: "etcd-cluster-1" - ETCD_INITIAL_CLUSTER: "etcd0=http://etcd0:2380" - ETCD_INITIAL_CLUSTER_STATE: "new" - ETCD_DATA_DIR: "/var/etcd/data" - ETCD_SNAPSHOT_CATCHUP_ENTRIES: 100 - ETCD_SNAPSHOT_COUNT: 50 - ETCD_COMPACTION_BATCH_LIMIT: 10 - ETCD_VERIFY: "all" - user: "${USER_ID:-root}:${GROUP_ID:-root}" - depends_on: - init: - condition: service_completed_successfully - ports: - - 12379:2379 - volumes: - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcd/data - - client: - image: 'etcd-client:${IMAGE_TAG:-latest}' - container_name: client - entrypoint: ["/opt/antithesis/entrypoint/entrypoint"] - user: "${USER_ID:-root}:${GROUP_ID:-root}" - environment: - ETCD_ROBUSTNESS_ENDPOINTS: "etcd0:2379" - ETCD_ROBUSTNESS_DATA_PATHS: "/var/etcddata0" - ETCD_ROBUSTNESS_REPORT_PATH: "/var/report" - depends_on: - etcd0: - condition: service_started - volumes: - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcddata0 - - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report diff --git a/tests/antithesis/config/docker-compose-3-node.yml b/tests/antithesis/config/docker-compose-3-node.yml deleted file mode 100644 index bc779af0de48..000000000000 --- a/tests/antithesis/config/docker-compose-3-node.yml +++ /dev/null @@ -1,124 +0,0 @@ ---- -services: - # This is needed for creating non-root data folders on host. - # By default, if the folders don't exist when mounting, compose creates them with root as owner. - # With root owner, accessing the WAL files from local tests will fail due to an unauthorized access error. - init: - image: 'docker.io/library/ubuntu:latest' - user: root - group_add: - - '${GROUP_ID:-root}' - volumes: - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcddata0 - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}1:/var/etcddata1 - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}2:/var/etcddata2 - - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report - command: - - /bin/sh - - -c - - | - rm -rf /var/etcddata0/* /var/etcddata1/* /var/etcddata2/* /var/report/* - chown -R ${USER_ID:-root}:${GROUP_ID:-root} /var/etcddata0 /var/etcddata1 /var/etcddata2 /var/report - - etcd0: - image: 'etcd-server:${IMAGE_TAG:-latest}' - container_name: etcd0 - hostname: etcd0 - environment: - ETCD_NAME: "etcd0" - ETCD_INITIAL_ADVERTISE_PEER_URLS: "http://etcd0:2380" - ETCD_LISTEN_PEER_URLS: "http://0.0.0.0:2380" - ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" - ETCD_ADVERTISE_CLIENT_URLS: "http://etcd0.etcd:2379" - ETCD_INITIAL_CLUSTER_TOKEN: "etcd-cluster-1" - ETCD_INITIAL_CLUSTER: "etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380" - ETCD_INITIAL_CLUSTER_STATE: "new" - ETCD_DATA_DIR: "/var/etcd/data" - ETCD_SNAPSHOT_CATCHUP_ENTRIES: 100 - ETCD_SNAPSHOT_COUNT: 50 - ETCD_COMPACTION_BATCH_LIMIT: 10 - ETCD_VERIFY: "all" - user: "${USER_ID:-root}:${GROUP_ID:-root}" - depends_on: - init: - condition: service_completed_successfully - ports: - - 12379:2379 - volumes: - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcd/data - - etcd1: - image: 'etcd-server:${IMAGE_TAG:-latest}' - container_name: etcd1 - hostname: etcd1 - environment: - ETCD_NAME: "etcd1" - ETCD_INITIAL_ADVERTISE_PEER_URLS: "http://etcd1:2380" - ETCD_LISTEN_PEER_URLS: "http://0.0.0.0:2380" - ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" - ETCD_ADVERTISE_CLIENT_URLS: "http://etcd1.etcd:2379" - ETCD_INITIAL_CLUSTER_TOKEN: "etcd-cluster-1" - ETCD_INITIAL_CLUSTER: "etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380" - ETCD_INITIAL_CLUSTER_STATE: "new" - ETCD_DATA_DIR: "/var/etcd/data" - ETCD_SNAPSHOT_CATCHUP_ENTRIES: 100 - ETCD_SNAPSHOT_COUNT: 50 - ETCD_COMPACTION_BATCH_LIMIT: 10 - ETCD_VERIFY: "all" - user: "${USER_ID:-root}:${GROUP_ID:-root}" - depends_on: - init: - condition: service_completed_successfully - ports: - - 22379:2379 - volumes: - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}1:/var/etcd/data - - etcd2: - image: 'etcd-server:${IMAGE_TAG:-latest}' - container_name: etcd2 - hostname: etcd2 - environment: - ETCD_NAME: "etcd2" - ETCD_INITIAL_ADVERTISE_PEER_URLS: "http://etcd2:2380" - ETCD_LISTEN_PEER_URLS: "http://0.0.0.0:2380" - ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" - ETCD_ADVERTISE_CLIENT_URLS: "http://etcd2.etcd:2379" - ETCD_INITIAL_CLUSTER_TOKEN: "etcd-cluster-1" - ETCD_INITIAL_CLUSTER: "etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380" - ETCD_INITIAL_CLUSTER_STATE: "new" - ETCD_DATA_DIR: "/var/etcd/data" - ETCD_SNAPSHOT_CATCHUP_ENTRIES: 100 - ETCD_SNAPSHOT_COUNT: 50 - ETCD_COMPACTION_BATCH_LIMIT: 10 - ETCD_VERIFY: "all" - user: "${USER_ID:-root}:${GROUP_ID:-root}" - depends_on: - init: - condition: service_completed_successfully - ports: - - 32379:2379 - volumes: - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}2:/var/etcd/data - - client: - image: 'etcd-client:${IMAGE_TAG:-latest}' - container_name: client - entrypoint: ["/opt/antithesis/entrypoint/entrypoint"] - user: "${USER_ID:-root}:${GROUP_ID:-root}" - environment: - ETCD_ROBUSTNESS_ENDPOINTS: "etcd0:2379,etcd1:2379,etcd2:2379" - ETCD_ROBUSTNESS_DATA_PATHS: "/var/etcddata0,/var/etcddata1,/var/etcddata2" - ETCD_ROBUSTNESS_REPORT_PATH: "/var/report" - depends_on: - etcd0: - condition: service_started - etcd1: - condition: service_started - etcd2: - condition: service_started - volumes: - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}0:/var/etcddata0 - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}1:/var/etcddata1 - - ${ETCD_ROBUSTNESS_DATA_PATH_PREFIX:-/tmp/etcddata}2:/var/etcddata2 - - ${ETCD_ROBUSTNESS_REPORT_PATH:-/tmp/etcdreport}:/var/report diff --git a/tests/antithesis/config/manifests/client.yaml b/tests/antithesis/config/manifests/client.yaml new file mode 100644 index 000000000000..042415b37133 --- /dev/null +++ b/tests/antithesis/config/manifests/client.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: etcd-client + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: etcd-client + template: + metadata: + labels: + app: etcd-client + spec: + containers: + - name: client + image: etcd-client:latest + imagePullPolicy: Never + command: ["/opt/antithesis/entrypoint/entrypoint"] + env: + - name: ETCD_ROBUSTNESS_ENDPOINTS + value: "etcd-0.etcd.default.svc.cluster.local:2379,etcd-1.etcd.default.svc.cluster.local:2379,etcd-2.etcd.default.svc.cluster.local:2379" + - name: ETCD_ROBUSTNESS_DATA_PATHS + value: "/var/etcddata/etcd-0,/var/etcddata/etcd-1,/var/etcddata/etcd-2" + - name: ETCD_ROBUSTNESS_REPORT_PATH + value: "/var/reports/report" + volumeMounts: + - name: data + mountPath: /var/etcddata + - name: report + mountPath: /var/reports + volumes: + - name: data + hostPath: + path: /tmp/etcddata + type: DirectoryOrCreate + - name: report + hostPath: + path: /tmp/etcdreport + type: DirectoryOrCreate diff --git a/tests/antithesis/config/manifests/default-etcd-3-replicas.yaml b/tests/antithesis/config/manifests/default-etcd-3-replicas.yaml index ceaeb8ede006..df74c34f1c51 100644 --- a/tests/antithesis/config/manifests/default-etcd-3-replicas.yaml +++ b/tests/antithesis/config/manifests/default-etcd-3-replicas.yaml @@ -91,13 +91,9 @@ spec: volumeMounts: - name: data mountPath: /var/etcd/data - volumeClaimTemplates: - - metadata: - name: data - spec: - storageClassName: standard - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 200Mi + subPathExpr: $(ETCD_NAME) + volumes: + - name: data + hostPath: + path: /tmp/etcddata + type: DirectoryOrCreate