diff --git a/cmd/config.go b/cmd/config.go index 90c5e4a..81f3f7a 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -38,12 +38,12 @@ type ScubaFeatureConfig struct { } type BucketNotificationsFeatureConfig struct { - Enabled bool `yaml:"enabled"` - TargetAuth struct { + Enabled bool `yaml:"enabled"` + DestinationAuth struct { Type string `yaml:"type"` Username string `yaml:"username"` Password string `yaml:"password"` - } `yaml:"targetAuth"` + } `yaml:"destinationAuth"` } type CloudserverConfig struct { @@ -150,6 +150,17 @@ func DefaultConfig() Config { LogLevel: "info", // Profile: "default", }, + Features: FeatureConfig{ + BucketNotifications: BucketNotificationsFeatureConfig{ + DestinationAuth: struct { + Type string `yaml:"type"` + Username string `yaml:"username"` + Password string `yaml:"password"` + }{ + Type: "none", + }, + }, + }, Cloudserver: CloudserverConfig{}, S3Metadata: MetadataConfig{ VFormat: Formatv1, diff --git a/cmd/configure.go b/cmd/configure.go index 55eab95..7e27aec 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -47,6 +47,7 @@ func configureEnv(cfg Config, envDir string) error { generateScubaConfig, generateS3MetadataConfig, generateScubaMetadataConfig, + generateKafkaConfig, } configDir := filepath.Join(envDir, "config") @@ -75,61 +76,34 @@ func generateCloudserverConfig(cfg Config, path string) error { func generateBackbeatConfig(cfg Config, path string) error { templates := []string{ - "Dockerfile.setup", - "setup.sh", - "setup-kafka-target.sh", - "config.notification.json", - "config.json", - "supervisord.conf", "env", + "supervisord.conf", + "config.json", + "config.notification.json", + "notificationCredentials.json", } - for _, tmpl := range templates { - templatePath := filepath.Join("templates", "backbeat", tmpl) - outputPath := filepath.Join(path, "backbeat", tmpl) - if err := renderTemplateToFile(getTemplates(), templatePath, cfg, outputPath); err != nil { - return fmt.Errorf("failed to render template %s: %w", tmpl, err) - } - } - return nil + return renderTemplates(cfg, "templates/backbeat", filepath.Join(path, "backbeat"), templates) } func generateVaultConfig(cfg Config, path string) error { - err := renderTemplateToFile(getTemplates(), "templates/vault/config.json", cfg, filepath.Join(path, "vault", "config.json")) - if err != nil { - return err - } - - err = renderTemplateToFile(getTemplates(), "templates/vault/create-management-account.sh", cfg, filepath.Join(path, "vault", "create-management-account.sh")) - if err != nil { - return err - } - - err = renderTemplateToFile(getTemplates(), "templates/vault/Dockerfile.setup", cfg, filepath.Join(path, "vault", "Dockerfile.setup")) - if err != nil { - return err - } - - err = renderTemplateToFile(getTemplates(), "templates/vault/management-creds.json", cfg, filepath.Join(path, "vault", "management-creds.json")) - if err != nil { - return err + templates := []string{ + "config.json", + "create-management-account.sh", + "Dockerfile.setup", + "management-creds.json", } - return nil + return renderTemplates(cfg, "templates/vault", filepath.Join(path, "vault"), templates) } func generateScubaConfig(cfg Config, path string) error { - err := renderTemplateToFile(getTemplates(), "templates/scuba/create-service-user.sh", cfg, filepath.Join(path, "scuba", "create-service-user.sh")) - if err != nil { - return err - } - - err = renderTemplateToFile(getTemplates(), "templates/scuba/Dockerfile.setup", cfg, filepath.Join(path, "scuba", "Dockerfile.setup")) - if err != nil { - return err + templates := []string{ + "config.json", + "create-service-user.sh", + "Dockerfile.setup", } - - return renderTemplateToFile(getTemplates(), "templates/scuba/config.json", cfg, filepath.Join(path, "scuba", "config.json")) + return renderTemplates(cfg, "templates/scuba", filepath.Join(path, "scuba"), templates) } func generateMetadataConfig(cfg MetadataConfig, path string) error { @@ -145,3 +119,16 @@ func generateScubaMetadataConfig(cfg Config, path string) error { cfgPath := filepath.Join(path, "metadata-scuba") return generateMetadataConfig(cfg.ScubaMetadata, cfgPath) } + +func generateKafkaConfig(cfg Config, path string) error { + templates := []string{ + "Dockerfile", + "setup.sh", + "server.backbeat.properties", + "server.destination.properties", + "config.properties", + "zookeeper.properties", + } + + return renderTemplates(cfg, "templates/kafka", filepath.Join(path, "kafka"), templates) +} diff --git a/cmd/util.go b/cmd/util.go index fe7b132..91f950b 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -48,6 +48,18 @@ func renderTemplateToFile(templates fs.FS, tmplPath string, data any, outPath st return nil } +func renderTemplates(cfg Config, srcDir, destDir string, templates []string) error { + templateFS := getTemplates() + for _, tmpl := range templates { + templatePath := filepath.Join(srcDir, tmpl) + outputPath := filepath.Join(destDir, tmpl) + if err := renderTemplateToFile(templateFS, templatePath, cfg, outputPath); err != nil { + return fmt.Errorf("failed to render template %s: %w", tmpl, err) + } + } + return nil +} + func getComposeProfiles(cfg Config) []string { profiles := []string{"base"} diff --git a/templates/backbeat/Dockerfile.setup b/templates/backbeat/Dockerfile.setup deleted file mode 100644 index e0652ed..0000000 --- a/templates/backbeat/Dockerfile.setup +++ /dev/null @@ -1,34 +0,0 @@ -FROM alpine:3.22 - -ARG KAFKA_VERSION=3.4.0 -ARG SCALA_VERSION=2.13 -ENV KAFKA_HOME=/opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} -ENV PATH="$PATH:$KAFKA_HOME/bin" - -RUN apk add --no-cache \ - bash \ - curl\ - jq \ - git \ - openjdk17 \ - netcat-openbsd \ - ca-certificates \ - && curl -fSL "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" -o /tmp/kafka.tgz && \ - tar -xzf /tmp/kafka.tgz -C /opt && \ - rm /tmp/kafka.tgz - -COPY --chmod=755 setup.sh / -COPY --chmod=755 setup-kafka-target.sh / - - -# ARG BASE_IMAGE - -# FROM ${BASE_IMAGE} - -# RUN apt-get update \ -# && apt-get install -y --no-install-recommends \ -# netcat - -# COPY --chmod=755 create-topics.sh /usr/local/bin/create-topics.sh - -# ENTRYPOINT ["/usr/local/bin/create-topics.sh"] diff --git a/templates/backbeat/config.notification.json b/templates/backbeat/config.notification.json index b8de011..f1d26c4 100644 --- a/templates/backbeat/config.notification.json +++ b/templates/backbeat/config.notification.json @@ -35,9 +35,16 @@ { "resource": "destination1", "type": "kafka", - "host": "localhost:9092", - "topic": "destination-topic", + "host": "127.0.0.1:9094", + "topic": "notifications", + {{ if eq .Features.BucketNotifications.DestinationAuth.Type "basic" }} + "auth": { + "type": "basic", + "credentialsFile": "notificationCredentials.json" + } + {{ else if eq .Features.BucketNotifications.DestinationAuth.Type "none" }} "auth": {} + {{ end }} } ] } diff --git a/templates/backbeat/notificationCredentials.json b/templates/backbeat/notificationCredentials.json new file mode 100644 index 0000000..a575e4d --- /dev/null +++ b/templates/backbeat/notificationCredentials.json @@ -0,0 +1,4 @@ +{ + "username": "{{ .Features.BucketNotifications.DestinationAuth.Username }}", + "password": "{{ .Features.BucketNotifications.DestinationAuth.Password }}" +} diff --git a/templates/backbeat/setup-kafka-target.sh b/templates/backbeat/setup-kafka-target.sh deleted file mode 100644 index 67d6e0f..0000000 --- a/templates/backbeat/setup-kafka-target.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -e - -# === Constants === -KAFKA_HOST=127.0.0.1 -KAFKA_PORT=9093 -KAFKA_BROKER="${KAFKA_HOST}:${KAFKA_PORT}" - -topic="destination-topic" - -# === Environment Echo === -echo "[setup] Kafka broker: $KAFKA_BROKER" -echo "[setup] Topics to create: $topic" -echo - -# === Wait for Kafka to be ready === -echo "[setup] Waiting for Kafka broker..." -until nc -z "$KAFKA_HOST" "$KAFKA_PORT"; do - echo "[setup] Kafka not ready, retrying in 2s..." - sleep 2 -done -echo "[setup] Kafka is ready!" -echo - -echo "[setup] Creating topic: $topic" -kafka-topics.sh \ - --create \ - --if-not-exists \ - --bootstrap-server "127.0.0.1:9093" \ - --topic "$topic" \ - --partitions 1 \ - --replication-factor 1 -echo "[setup] Topic '$topic' created (or already exists)." -echo diff --git a/templates/backbeat/setup.sh b/templates/backbeat/setup.sh deleted file mode 100644 index 1e2deb4..0000000 --- a/templates/backbeat/setup.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -set -e - -# === Constants === -KAFKA_HOST=127.0.0.1 -KAFKA_PORT=9092 -KAFKA_BROKER="${KAFKA_HOST}:${KAFKA_PORT}" - -# List of topics to create -LIFECYCLE_TOPICS="backbeat-lifecycle-bucket-tasks backbeat-lifecycle-object-tasks" -NOTIFICATION_TOPICS="backbeat-bucket-notification" - -function create_topics() { - local topics="$1" - for topic in $topics; do - echo "[setup] Creating topic: $topic" - kafka-topics.sh \ - --create \ - --if-not-exists \ - --bootstrap-server "$KAFKA_BROKER" \ - --topic "$topic" \ - --partitions 1 \ - --replication-factor 1 - echo "[setup] Topic '$topic' created (or already exists)." - echo - done -} - -# === Environment Echo === -echo "[setup] Kafka broker: $KAFKA_BROKER" -echo "[setup] Topics to create: $TOPICS" -echo - -# === Wait for Kafka to be ready === -echo "[setup] Waiting for Kafka broker..." -until nc -z "$KAFKA_HOST" "$KAFKA_PORT"; do - echo "[setup] Kafka not ready, retrying in 2s..." - sleep 2 -done -echo "[setup] Kafka is ready!" -echo - -# === Create topics === -echo "[setup] Creating lifecycle topics..." -create_topics "$LIFECYCLE_TOPICS" - -echo "[setup] Creating notification topics..." -create_topics "$NOTIFICATION_TOPICS" - -echo "[setup] Creating Zookeeper paths..." -zookeeper-shell.sh localhost:2181/backbeat <