etcd-io · serathius · Mar 28, 2026
diff --git a/.github/workflows/antithesis-test.yml b/.github/workflows/antithesis-test.yml
@@ -78,7 +78,7 @@ jobs:
       - name: Run Antithesis Tests
         uses: antithesishq/antithesis-trigger-action@f6221e2ba819fe0ac3e36bd67a281fa439a03fba # v0.10
         with:
-          notebook_name: etcd
+          notebook_name: etcd_k8s
           tenant: linuxfoundation
           username: ${{ secrets.ANTITHESIS_WEBHOOK_USERNAME }}
           password: ${{ secrets.ANTITHESIS_WEBHOOK_PASSWORD }}
@@ -89,5 +89,5 @@ jobs:
           email_recipients: ${{ inputs.email || 'siarkowicz@google.com' }}
           test_name: ${{ inputs.test || 'etcd nightly antithesis run' }}
           additional_parameters: |-
-            custom.duration = ${{ inputs.duration || 12 }}
+            antithesis.duration = ${{ inputs.duration || 12 }}
             antithesis.source = ${{ inputs.etcd_ref || 'main' }}
diff --git a/.github/workflows/antithesis-verify.yml b/.github/workflows/antithesis-verify.yml
@@ -1,5 +1,5 @@
 ---
-name: Verify Antithesis Docker Compose Pipeline
+name: Antithesis Verify
 
 permissions:
   contents: read
@@ -12,61 +12,35 @@ on:
       - 'tests/antithesis/**'
       - '.github/workflows/antithesis-verify.yml'
   pull_request:
+    branches:
+      - main
     paths:
       - 'tests/antithesis/**'
       - '.github/workflows/antithesis-verify.yml'
 
 jobs:
-  test-docker-compose:
-    strategy:
-      matrix:
-        node-count: [1, 3]
-    name: Test ${{ matrix.node-count }}-node cluster
+  test:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-
-      - name: Build etcd-server and etcd-client images
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Build images
         run: |
           make -C tests/antithesis antithesis-build-etcd-image
           make -C tests/antithesis antithesis-build-client-docker-image
 
-      - name: Run docker-compose up
+      - name: Set up Kubernetes cluster
         working-directory: ./tests/antithesis
-        run: |
-          make antithesis-docker-compose-up CFG_NODE_COUNT=${{ matrix.node-count }} &
-
-      - name: Check for healthy cluster
-        working-directory: ./tests/antithesis
-        run: |
-          timeout=120
-          interval=10
-          end_time=$(( $(date +%s) + timeout ))
-
-          while [ $(date +%s) -lt $end_time ]; do
-            # The client container might not be running yet, so ignore errors from docker compose logs
-            if docker compose -f config/docker-compose-${{ matrix.node-count }}-node.yml logs client 2>/dev/null | grep -q "Client \[entrypoint\]: cluster is healthy!"; then
-              echo "Cluster is healthy!"
-              exit 0
-            fi
-            echo "Waiting for cluster to become healthy..."
-            sleep $interval
-          done
-
-          echo "Cluster did not become healthy in ${timeout} seconds."
-          docker compose -f config/docker-compose-${{ matrix.node-count }}-node.yml logs
-          exit 1
+        run: make antithesis-k8s-up
 
       - name: Run traffic
         working-directory: ./tests/antithesis
-        run: make antithesis-run-container-traffic CFG_NODE_COUNT=${{ matrix.node-count }}
+        run: make antithesis-run-k8s-traffic
 
       - name: Run validation
         working-directory: ./tests/antithesis
-        run: make antithesis-run-container-validation CFG_NODE_COUNT=${{ matrix.node-count }}
+        run: make antithesis-run-k8s-validation
 
       - name: Clean up
         if: always()
         working-directory: ./tests/antithesis
-        run: make antithesis-clean CFG_NODE_COUNT=${{ matrix.node-count }}
+        run: make antithesis-clean
diff --git a/tests/antithesis/Makefile b/tests/antithesis/Makefile
@@ -45,41 +45,41 @@ antithesis-build-etcd-image-main: REF=main
 antithesis-build-etcd-image-main: antithesis-build-etcd-image
 
 .PHONY: antithesis-build-config-image
-antithesis-build-config-image: validate-node-count
+antithesis-build-config-image:
 	docker build -f config/Dockerfile config -t etcd-config:latest \
-		--build-arg IMAGE_TAG=$(IMAGE_TAG) \
-		--build-arg NODE_COUNT=$(CFG_NODE_COUNT)
+		--build-arg IMAGE_TAG=$(IMAGE_TAG)
 
-.PHONY: antithesis-docker-compose-up
-antithesis-docker-compose-up: validate-node-count
-	export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \
-		docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml up
+.PHONY: antithesis-k8s-up
+antithesis-k8s-up: check-k8s-tools
+	kind create cluster --name etcd-antithesis
+	kind load docker-image etcd-client:latest --name etcd-antithesis
+	kind load docker-image etcd-server:latest --name etcd-antithesis
+	kubectl apply -f config/manifests
+	kubectl wait --for=condition=ready pod -l app=etcd --timeout=120s
+	kubectl wait --for=condition=ready pod -l app=etcd-client --timeout=120s
 
-.PHONY: antithesis-run-container-traffic
-antithesis-run-container-traffic: validate-node-count
-	export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \
-		docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml exec client /opt/antithesis/test/v1/robustness/singleton_driver_traffic
+.PHONY: antithesis-run-k8s-traffic
+antithesis-run-k8s-traffic:
+	kubectl exec deployment/etcd-client -- /opt/antithesis/test/v1/robustness/singleton_driver_traffic
 
-.PHONY: antithesis-run-container-validation
-antithesis-run-container-validation: validate-node-count
-	export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \
-		docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml exec client /opt/antithesis/test/v1/robustness/finally_validation
+.PHONY: antithesis-run-k8s-validation
+antithesis-run-k8s-validation:
+	kubectl exec deployment/etcd-client -- /opt/antithesis/test/v1/robustness/finally_validation
 
 .PHONY: antithesis-run-local-traffic
 antithesis-run-local-traffic:
-	export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata0,/tmp/etcddata1,/tmp/etcddata2 && export ETCD_ROBUSTNESS_REPORT_PATH=report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \
+	export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata/etcd-0,/tmp/etcddata/etcd-1,/tmp/etcddata/etcd-2 && export ETCD_ROBUSTNESS_REPORT_PATH=reports/report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \
 		go run --race ./test-template/robustness/traffic/main.go
 
 .PHONY: antithesis-run-local-validation
 antithesis-run-local-validation:
-	export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata0,/tmp/etcddata1,/tmp/etcddata2 && export ETCD_ROBUSTNESS_REPORT_PATH=report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \
+	export ETCD_ROBUSTNESS_DATA_PATHS=/tmp/etcddata/etcd-0,/tmp/etcddata/etcd-1,/tmp/etcddata/etcd-2 && export ETCD_ROBUSTNESS_REPORT_PATH=reports/report && export ETCD_ROBUSTNESS_ENDPOINTS=127.0.0.1:12379,127.0.0.1:22379,127.0.0.1:32379 && \
 		go run --race ./test-template/robustness/finally/main.go
 
 .PHONY: antithesis-clean
-antithesis-clean: validate-node-count
-	export USER_ID=$(USER_ID) && export GROUP_ID=$(GROUP_ID) && \
-		docker compose -f config/docker-compose-$(CFG_NODE_COUNT)-node.yml down --remove-orphans
-	rm -rf /tmp/etcddata0 /tmp/etcddata1 /tmp/etcddata2 /tmp/etcdreport
+antithesis-clean: check-k8s-tools
+	kind delete cluster --name etcd-antithesis
+	rm -rf /tmp/etcddata /tmp/etcdreport
 
 .PHONY: validate-node-count
 validate-node-count:

diff --git a/tests/antithesis/README.md b/tests/antithesis/README.md
@@ -17,7 +17,7 @@ For more details on robustness tests, see the [robustness directory](../robustne
 ## Antithesis Setup
 
 The setup consists of a 3-node etcd cluster and a client container, orchestrated
-via [Docker Compose](https://antithesis.com/docs/getting_started/setup/).
+via [Kubernetes](https://antithesis.com/docs/getting_started/setup_k8s/).
 
 During the etcd Antithesis test suite the etcd server is built with the following patches:
 
@@ -52,11 +52,18 @@ in the following way:
 [Singleton Driver Command]: https://antithesis.com/docs/test_templates/test_composer_reference/#singleton-driver
 [Finally Command]: https://antithesis.com/docs/test_templates/test_composer_reference/#finally-command
 
-# Running tests with docker compose
+## Running tests locally with Kubernetes
+
+### Prerequisites
+
+Please make sure that you have the following tools installed on your local:
+
+* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
+* [kind](https://kind.sigs.k8s.io/docs/user/quick-start#installation)
 
 ## Quickstart
 
-### 1. Build and Tag the Docker Image
+### 1. Build and Tag the Docker Images
 
 Run this command from the `antithesis/test-template` directory:
 
@@ -71,140 +78,42 @@ Both commands build etcd-server and etcd-client from the current branch. To buil
 make antithesis-build-etcd-image REF=${GIT_REF} 
 ```
 
-### 2. (Optional) Check the Image Locally
-
-You can verify your new image is built:
-
-```bash
-docker images | grep etcd-client
-```
-
-It should show something like:
-
-```
-etcd-client        latest    <IMAGE_ID>    <DATE>
-```
-
-### 3. Use in Docker Compose
+#### 2. Set up the Kubernetes cluster
 
 Run the following command from the root directory for Antithesis tests (`tests/antithesis`):
 
 ```bash
-make antithesis-docker-compose-up
+make antithesis-k8s-up
 ```
 
-The command uses the etcd client and server images built from step 1.
+This command will:
 
-The client will continuously check the health of the etcd nodes and print logs similar to:
+* Create a local `kind` cluster named `etcd-antithesis`.
+* Load the built `etcd-client` and `etcd-server` images into the cluster.
+* Deploy the Kubernetes manifests from `config/manifests`.
 
-```
-[+] Running 4/4
- ✔ Container etcd0   Created                                                                                                                                                 0.0s 
- ✔ Container etcd2   Created                                                                                                                                                 0.0s 
- ✔ Container etcd1   Created                                                                                                                                                 0.0s 
- ✔ Container client  Recreated                                                                                                                                               0.1s 
-Attaching to client, etcd0, etcd1, etcd2
-etcd2   | {"level":"info","ts":"2025-04-14T07:23:25.134294Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ADVERTISE_CLIENT_URLS","variable-value":"http://etcd2.etcd:2379"}
-etcd2   | {"level":"info","ts":"2025-04-14T07:23:25.138501Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_ADVERTISE_PEER_URLS","variable-value":"http://etcd2:2380"}
-etcd2   | {"level":"info","ts":"2025-04-14T07:23:25.138646Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER","variable-value":"etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380"}
-etcd0   | {"level":"info","ts":"2025-04-14T07:23:25.138434Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_ADVERTISE_CLIENT_URLS","variable-value":"http://etcd0.etcd:2379"}
-etcd0   | {"level":"info","ts":"2025-04-14T07:23:25.138582Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_ADVERTISE_PEER_URLS","variable-value":"http://etcd0:2380"}
-etcd0   | {"level":"info","ts":"2025-04-14T07:23:25.138592Z","caller":"flags/flag.go:113","msg":"recognized and used environment variable","variable-name":"ETCD_INITIAL_CLUSTER","variable-value":"etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380"}
-
-...
-...
-(skipping some repeated logs for brevity)
-...
-...
-
-etcd2   | {"level":"info","ts":"2025-04-14T07:23:25.484698Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"}
-etcd1   | {"level":"info","ts":"2025-04-14T07:23:25.484092Z","caller":"embed/serve.go:210","msg":"serving client traffic insecurely; this is strongly discouraged!","traffic":"grpc+http","address":"[::]:2379"}
-etcd0   | {"level":"info","ts":"2025-04-14T07:23:25.484563Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"}
-etcd2   | {"level":"info","ts":"2025-04-14T07:23:25.485101Z","caller":"v3rpc/health.go:61","msg":"grpc service status changed","service":"","status":"SERVING"}
-etcd1   | {"level":"info","ts":"2025-04-14T07:23:25.484130Z","caller":"etcdmain/main.go:44","msg":"notifying init daemon"}
-etcd2   | {"level":"info","ts":"2025-04-14T07:23:25.485782Z","caller":"embed/serve.go:210","msg":"serving client traffic insecurely; this is strongly discouraged!","traffic":"grpc+http","address":"[::]:2379"}
-etcd1   | {"level":"info","ts":"2025-04-14T07:23:25.484198Z","caller":"etcdmain/main.go:50","msg":"successfully notified init daemon"}
-client  | Client [entrypoint]: starting...
-client  | Client [entrypoint]: checking cluster health...
-client  | Client [entrypoint]: connection successful with etcd0
-client  | Client [entrypoint]: connection successful with etcd1
-client  | Client [entrypoint]: connection successful with etcd2
-client  | Client [entrypoint]: cluster is healthy!
-```
+The client pod will continuously check the health of the etcd nodes and wait for the cluster to be healthy.
 
-And it will stay running indefinitely.
+#### 3. Running the tests
 
-### 4. Running the tests
+To run the tests locally against the deployed cluster:
 
 ```bash
-make antithesis-run-container-traffic
-make antithesis-run-container-validation
+make antithesis-run-k8s-traffic
+make antithesis-run-k8s-validation
 ```
 
-Alternatively, with the etcd cluster from step 3, to run the tests locally without rebuilding the client image:
-
-```bash
-make antithesis-run-local-traffic
-make antithesis-run-local-validation
-```
+#### 4. Prepare for next run
 
-### 5. Prepare for next run
-
-Unfortunatelly robustness tests don't support running on non empty database.
+Unfortunately robustness tests don't support running on a non-empty database.
 So for now you need to cleanup the storage before repeating the run or you will get "non empty database at start, required by model used for linearizability validation" error.
 
 ```bash
 make antithesis-clean
 ```
 
-## Troubleshooting
-
-- **Image Pull Errors**: If Docker can’t pull `etcd-client:latest`, make sure you built it locally (see the “Build and Tag” step) or push it to a registry that Compose can access.
-
-# Running Tests with Kubernetes (WIP)
-
-## Prerequisites
-
-Please make sure that you have the following tools installed on your local:
-
-- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
-- [kind](https://kind.sigs.k8s.io/docs/user/quick-start#installation)
-
-## Testing locally
-
-### Setting up the cluster and deploying the images
-
-#### 1. Ensure your access to a test kubernetes cluster
-
-You can use `kind` to create a local cluster to deploy the etcd-server and test client.  Once you have `kind` installed, you can use the following command to create a local cluster:
-
-```bash
-kind create cluster
-```
-
-Alternatively, you can use any existing kubernetes cluster you have access to.
-
-#### 2. Build and load the images
-
-Please [build the client and server images](#1-build-and-tag-the-docker-image) first. Then load the images into the `kind` cluster:
+This command will delete the `kind` cluster and remove the local data directories.
 
-If you use `kind`, the cluster will need to have access to the images using the following commands:
+### Troubleshooting
 
-```bash
-kind load docker-image etcd-client:latest
-kind load docker-image etcd-server:latest
-```
-
-If you use something other than `kind`, please make sure the images are accessible to your cluster. This might involve pushing the images to a container registry that your cluster can pull from.
-
-#### 3. Deploy the kubernetes manifests
-
-```bash
-kubectl apply -f ./config/manifests
-```
-
-### Tearing down the cluster
-
-```bash
-kind delete cluster --name kind
-```
+* **Image Pull Errors**: If Kubernetes can’t pull `etcd-client:latest`, make sure you built it locally and that it was successfully loaded into the `kind` cluster.
diff --git a/tests/antithesis/config/Dockerfile b/tests/antithesis/config/Dockerfile
@@ -1,12 +1,8 @@
-ARG GO_VERSION=1.25.5
-
-FROM golang:$GO_VERSION AS build
-RUN go install github.com/a8m/envsubst/cmd/envsubst@v1.4.3
-
-ARG IMAGE_TAG
-ARG NODE_COUNT
-COPY docker-compose-${NODE_COUNT}-node.yml /docker-compose.yml.template
-RUN IMAGE_TAG=${IMAGE_TAG} cat /docker-compose.yml.template | envsubst > /docker-compose.yml
+FROM alpine AS build
+ARG IMAGE_TAG=latest
+COPY manifests/ /manifests/
+RUN sed -i "s/etcd-client:latest/etcd-client:${IMAGE_TAG}/g" /manifests/client.yaml && \
+    sed -i "s/etcd-server:latest/etcd-server:${IMAGE_TAG}/g" /manifests/default-etcd-3-replicas.yaml
 
 FROM scratch
-COPY --from=build /docker-compose.yml /docker-compose.yml
+COPY --from=build /manifests/ /manifests/