feat: Support VM HA

.github/workflows/build.yml

@@ -14,6 +14,8 @@ jobs:
         with:
           go-version: 1.19.3
 
+      - run: sudo apt-get update && sudo apt-get install libaio-dev libsanlock-dev
+
       - run: make test
 
       - uses: codecov/codecov-action@v2

.github/workflows/release.yml

@@ -23,6 +23,30 @@ jobs:
           username: smartxrocks
           password: ${{ secrets.DOCKERHUB_PUSH_TOKEN }}
 
+      - id: build_lockspace_initializer
+        uses: docker/build-push-action@v2
+        with:
+          file: build/lockspace-initializer/Dockerfile
+          tags: smartxworks/lockspace-initializer:${{ steps.get_version.outputs.version }}
+          platforms: linux/amd64,linux/arm64
+          push: true
+
+      - id: build_lockspace_attacher
+        uses: docker/build-push-action@v2
+        with:
+          file: build/lockspace-attacher/Dockerfile
+          tags: smartxworks/lockspace-attacher:${{ steps.get_version.outputs.version }}
+          platforms: linux/amd64,linux/arm64
+          push: true
+
+      - id: build_lockspace_detector
+        uses: docker/build-push-action@v2
+        with:
+          file: build/lockspace-detector/Dockerfile
+          tags: smartxworks/lockspace-detector:${{ steps.get_version.outputs.version }}
+          platforms: linux/amd64,linux/arm64
+          push: true
+
       - id: build_virt_prerunner
         uses: docker/build-push-action@v2
         with:
@@ -34,7 +58,11 @@ jobs:
       - uses: docker/build-push-action@v2
         with:
           file: build/virt-controller/Dockerfile
-          build-args: PRERUNNER_IMAGE=smartxworks/virt-prerunner:${{ steps.get_version.outputs.version }}@${{ steps.build_virt_prerunner.outputs.digest }}
+          build-args: |
+            PRERUNNER_IMAGE=smartxworks/virt-prerunner:${{ steps.get_version.outputs.version }}@${{ steps.build_virt_prerunner.outputs.digest }}
+            INITIALIZER_IMAGE=smartxworks/lockspace-initializer:${{ steps.get_version.outputs.version }}@${{ steps.build_lockspace_initializer.outputs.digest }}
+            ATTACHER_IMAGE=smartxworks/lockspace-attacher:${{ steps.get_version.outputs.version }}@${{ steps.build_lockspace_attacher.outputs.digest }}
+            DETECTOR_IMAGE=smartxworks/lockspace-detector:${{ steps.get_version.outputs.version }}@${{ steps.build_lockspace_detector.outputs.digest }}
           tags: smartxworks/virt-controller:${{ steps.get_version.outputs.version }}
           platforms: linux/amd64,linux/arm64
           push: true

build/lockspace-attacher/Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:1.19-alpine AS builder
+
+RUN apk add --no-cache gcc musl-dev libaio-dev
+RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+WORKDIR /workspace
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/lockspace-attacher/main.go
+
+FROM alpine
+
+RUN apk add --no-cache sanlock --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+COPY --from=builder /workspace/main /usr/bin/lockspace-attacher
+ENTRYPOINT ["lockspace-attacher"]

build/lockspace-detector/Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:1.19-alpine AS builder
+
+RUN apk add --no-cache gcc musl-dev libaio-dev
+RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+WORKDIR /workspace
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/lockspace-detector/main.go
+
+FROM alpine
+
+RUN apk add --no-cache sanlock --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+COPY --from=builder /workspace/main /usr/bin/lockspace-detector
+ENTRYPOINT ["lockspace-detector"]

build/lockspace-initializer/Dockerfile

@@ -0,0 +1,21 @@
+FROM golang:1.19-alpine AS builder
+
+RUN apk add --no-cache gcc musl-dev libaio-dev
+RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+WORKDIR /workspace
+
+COPY go.mod go.mod
+COPY go.sum go.sum
+RUN go mod download
+
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/lockspace-initializer/main.go
+
+FROM alpine
+
+RUN apk add --no-cache sanlock --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+
+COPY --from=builder /workspace/main /usr/bin/lockspace-initializer
+ENTRYPOINT ["lockspace-initializer"]

build/virt-controller/Dockerfile

@@ -14,6 +14,12 @@ FROM alpine
 
 ARG PRERUNNER_IMAGE
 ENV PRERUNNER_IMAGE=$PRERUNNER_IMAGE
+ARG DETECTOR_IMAGE
+ENV DETECTOR_IMAGE=$DETECTOR_IMAGE
+ARG ATTACHER_IMAGE
+ENV ATTACHER_IMAGE=$ATTACHER_IMAGE
+ARG INITIALIZER_IMAGE
+ENV INITIALIZER_IMAGE=$INITIALIZER_IMAGE
 
 COPY --from=builder /workspace/main /usr/bin/virt-controller
 ENTRYPOINT ["virt-controller"]

build/virt-prerunner/Dockerfile

@@ -1,6 +1,7 @@
 FROM golang:1.19-alpine AS builder
 
-RUN apk add --no-cache gcc musl-dev
+RUN apk add --no-cache gcc musl-dev libaio-dev
+RUN apk add --no-cache sanlock-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
 
 WORKDIR /workspace
 
@@ -14,7 +15,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build go build -a cmd/virt-prerunn
 
 FROM alpine
 
-RUN apk add --no-cache curl screen dnsmasq cdrkit iptables iproute2 qemu-virtiofsd dpkg util-linux s6-overlay nmap-ncat
+RUN apk add --no-cache curl screen dnsmasq cdrkit iptables iproute2 qemu-virtiofsd dpkg util-linux tini nmap-ncat
+RUN apk add --no-cache sanlock --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
 
 RUN set -eux; \
     mkdir /var/lib/cloud-hypervisor; \
@@ -34,19 +36,9 @@ RUN set -eux; \
     chmod +x /usr/bin/cloud-hypervisor; \
     chmod +x /usr/bin/ch-remote
 
-COPY build/virt-prerunner/cloud-hypervisor-type /etc/s6-overlay/s6-rc.d/cloud-hypervisor/type
-COPY build/virt-prerunner/cloud-hypervisor-run.sh /etc/s6-overlay/s6-rc.d/cloud-hypervisor/run
-COPY build/virt-prerunner/cloud-hypervisor-finish.sh /etc/s6-overlay/s6-rc.d/cloud-hypervisor/finish
-RUN touch /etc/s6-overlay/s6-rc.d/user/contents.d/cloud-hypervisor
-
 COPY --from=builder /workspace/main /usr/bin/virt-prerunner
-COPY build/virt-prerunner/virt-prerunner-type /etc/s6-overlay/s6-rc.d/virt-prerunner/type
-COPY build/virt-prerunner/virt-prerunner-up /etc/s6-overlay/s6-rc.d/virt-prerunner/up
-COPY build/virt-prerunner/virt-prerunner-run.sh /etc/s6-overlay/scripts/virt-prerunner-run.sh
-RUN touch /etc/s6-overlay/s6-rc.d/user/contents.d/virt-prerunner
-ENV S6_BEHAVIOUR_IF_STAGE2_FAILS=2
 
-ENTRYPOINT ["/init"]
+ENTRYPOINT ["tini", "-s", "-g", "--", "/usr/bin/virt-prerunner"]
 
 COPY build/virt-prerunner/iptables-wrapper /sbin/iptables-wrapper
 RUN update-alternatives --install /sbin/iptables iptables /sbin/iptables-wrapper 100

cmd/lockspace-attacher/main.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strconv"
+	"syscall"
+
+	"github.com/namsral/flag"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	"github.com/smartxworks/virtink/pkg/sanlock"
+)
+
+func main() {
+	var lockspaceName string
+	var nodeName string
+	flag.StringVar(&lockspaceName, "lockspace-name", "", "")
+	flag.StringVar(&nodeName, "node-name", "", "")
+	flag.Parse()
+
+	cfg, err := clientcmd.BuildConfigFromFlags("", "")
+	if err != nil {
+		log.Fatalf("failed to build kubeconfig: %s", err)
+	}
+	kubeClient, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		log.Fatalf("failed to create Kubernetes client: %s", err)
+	}
+
+	hostID, err := getHostID(kubeClient, nodeName)
+	if err != nil {
+		log.Fatalf("failed to get host ID: %s", err)
+
+	}
+
+	leaseFilePath := filepath.Join("/var/lib/sanlock", lockspaceName, "leases")
+	if err := sanlock.AcquireDeltaLease(lockspaceName, leaseFilePath, hostID); err != nil {
+		log.Fatalf("failed to acquire delta lease: %s", err)
+	}
+	log.Println("succeeded to acquire delta lease")
+
+	stop := make(chan struct{})
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-c
+		close(stop)
+		<-c
+		os.Exit(1)
+	}()
+
+	<-stop
+	if err := sanlock.ReleaseDeltaLease(lockspaceName, leaseFilePath, hostID); err != nil {
+		if err != sanlock.ENOENT {
+			log.Fatalf("failed to release delta lease: %s", err)
+		}
+	}
+	if err := umountLeaseVolume(lockspaceName); err != nil {
+		log.Fatalf("failed to umont lease volume: %s", err)
+	}
+}
+
+//+kubebuilder:rbac:groups="",resources=nodes,verbs=get
+
+func getHostID(client *kubernetes.Clientset, nodeName string) (uint64, error) {
+	node, err := client.CoreV1().Nodes().Get(context.Background(), nodeName, metav1.GetOptions{})
+	if err != nil {
+		return 0, fmt.Errorf("get node: %s", err)
+	}
+
+	if id, exist := node.Annotations["virtink.smartx.com/sanlock-host-id"]; exist {
+		if hostID, err := strconv.ParseUint(id, 10, 64); err != nil {
+			return 0, fmt.Errorf("parse uint: %s", err)
+		} else {
+			return hostID, nil
+		}
+	}
+	return 0, fmt.Errorf("sanlock host %s ID not found", nodeName)
+}
+
+func umountLeaseVolume(lockspace string) error {
+	leaseFileDir := filepath.Join("/var/lib/sanlock", lockspace)
+	output, err := exec.Command("sh", "-c", fmt.Sprintf("mount | grep '%s'", leaseFileDir)).CombinedOutput()
+	if err != nil && string(output) == "" {
+		return nil
+	}
+
+	if _, err := exec.Command("umount", leaseFileDir).CombinedOutput(); err != nil {
+		return fmt.Errorf("unmount volume: %s", err)
+	}
+	return nil
+}