Add bucket notifications deployment supporting basic auth and no auth

cmd/config.go

@@ -38,12 +38,12 @@ type ScubaFeatureConfig struct {
 }
 
 type BucketNotificationsFeatureConfig struct {
-	Enabled    bool `yaml:"enabled"`
-	TargetAuth struct {
+	Enabled         bool `yaml:"enabled"`
+	DestinationAuth struct {
 		Type     string `yaml:"type"`
 		Username string `yaml:"username"`
 		Password string `yaml:"password"`
-	} `yaml:"targetAuth"`
+	} `yaml:"destinationAuth"`
 }
 
 type CloudserverConfig struct {
@@ -150,6 +150,17 @@ func DefaultConfig() Config {
 			LogLevel: "info",
 			// Profile:  "default",
 		},
+		Features: FeatureConfig{
+			BucketNotifications: BucketNotificationsFeatureConfig{
+				DestinationAuth: struct {
+					Type     string `yaml:"type"`
+					Username string `yaml:"username"`
+					Password string `yaml:"password"`
+				}{
+					Type: "none",
+				},
+			},
+		},
 		Cloudserver: CloudserverConfig{},
 		S3Metadata: MetadataConfig{
 			VFormat: Formatv1,

cmd/configure.go

@@ -47,6 +47,7 @@ func configureEnv(cfg Config, envDir string) error {
 		generateScubaConfig,
 		generateS3MetadataConfig,
 		generateScubaMetadataConfig,
+		generateKafkaConfig,
 	}
 
 	configDir := filepath.Join(envDir, "config")
@@ -75,61 +76,34 @@ func generateCloudserverConfig(cfg Config, path string) error {
 
 func generateBackbeatConfig(cfg Config, path string) error {
 	templates := []string{
-		"Dockerfile.setup",
-		"setup.sh",
-		"setup-kafka-target.sh",
-		"config.notification.json",
-		"config.json",
-		"supervisord.conf",
 		"env",
+		"supervisord.conf",
+		"config.json",
+		"config.notification.json",
+		"notificationCredentials.json",
 	}
 
-	for _, tmpl := range templates {
-		templatePath := filepath.Join("templates", "backbeat", tmpl)
-		outputPath := filepath.Join(path, "backbeat", tmpl)
-		if err := renderTemplateToFile(getTemplates(), templatePath, cfg, outputPath); err != nil {
-			return fmt.Errorf("failed to render template %s: %w", tmpl, err)
-		}
-	}
-	return nil
+	return renderTemplates(cfg, "templates/backbeat", filepath.Join(path, "backbeat"), templates)
 }
 
 func generateVaultConfig(cfg Config, path string) error {
-	err := renderTemplateToFile(getTemplates(), "templates/vault/config.json", cfg, filepath.Join(path, "vault", "config.json"))
-	if err != nil {
-		return err
-	}
-
-	err = renderTemplateToFile(getTemplates(), "templates/vault/create-management-account.sh", cfg, filepath.Join(path, "vault", "create-management-account.sh"))
-	if err != nil {
-		return err
-	}
-
-	err = renderTemplateToFile(getTemplates(), "templates/vault/Dockerfile.setup", cfg, filepath.Join(path, "vault", "Dockerfile.setup"))
-	if err != nil {
-		return err
-	}
-
-	err = renderTemplateToFile(getTemplates(), "templates/vault/management-creds.json", cfg, filepath.Join(path, "vault", "management-creds.json"))
-	if err != nil {
-		return err
+	templates := []string{
+		"config.json",
+		"create-management-account.sh",
+		"Dockerfile.setup",
+		"management-creds.json",
 	}
 
-	return nil
+	return renderTemplates(cfg, "templates/vault", filepath.Join(path, "vault"), templates)
 }
 
 func generateScubaConfig(cfg Config, path string) error {
-	err := renderTemplateToFile(getTemplates(), "templates/scuba/create-service-user.sh", cfg, filepath.Join(path, "scuba", "create-service-user.sh"))
-	if err != nil {
-		return err
-	}
-
-	err = renderTemplateToFile(getTemplates(), "templates/scuba/Dockerfile.setup", cfg, filepath.Join(path, "scuba", "Dockerfile.setup"))
-	if err != nil {
-		return err
+	templates := []string{
+		"config.json",
+		"create-service-user.sh",
+		"Dockerfile.setup",
 	}
-
-	return renderTemplateToFile(getTemplates(), "templates/scuba/config.json", cfg, filepath.Join(path, "scuba", "config.json"))
+	return renderTemplates(cfg, "templates/scuba", filepath.Join(path, "scuba"), templates)
 }
 
 func generateMetadataConfig(cfg MetadataConfig, path string) error {
@@ -145,3 +119,16 @@ func generateScubaMetadataConfig(cfg Config, path string) error {
 	cfgPath := filepath.Join(path, "metadata-scuba")
 	return generateMetadataConfig(cfg.ScubaMetadata, cfgPath)
 }
+
+func generateKafkaConfig(cfg Config, path string) error {
+	templates := []string{
+		"Dockerfile",
+		"setup.sh",
+		"server.backbeat.properties",
+		"server.destination.properties",
+		"config.properties",
+		"zookeeper.properties",
+	}
+
+	return renderTemplates(cfg, "templates/kafka", filepath.Join(path, "kafka"), templates)
+}

cmd/util.go

@@ -48,6 +48,18 @@ func renderTemplateToFile(templates fs.FS, tmplPath string, data any, outPath st
 	return nil
 }
 
+func renderTemplates(cfg Config, srcDir, destDir string, templates []string) error {
+	templateFS := getTemplates()
+	for _, tmpl := range templates {
+		templatePath := filepath.Join(srcDir, tmpl)
+		outputPath := filepath.Join(destDir, tmpl)
+		if err := renderTemplateToFile(templateFS, templatePath, cfg, outputPath); err != nil {
+			return fmt.Errorf("failed to render template %s: %w", tmpl, err)
+		}
+	}
+	return nil
+}
+
 func getComposeProfiles(cfg Config) []string {
 	profiles := []string{"base"}
 

templates/backbeat/config.notification.json

@@ -35,9 +35,16 @@
                 {
                     "resource": "destination1",
                     "type": "kafka",
-                    "host": "localhost:9092",
-                    "topic": "destination-topic",
+                    "host": "127.0.0.1:9094",
+                    "topic": "notifications",
+                    {{ if eq .Features.BucketNotifications.DestinationAuth.Type "basic" }}
+                    "auth": {
+                        "type": "basic",
+                        "credentialsFile": "notificationCredentials.json"
+                    }
+                    {{ else if eq .Features.BucketNotifications.DestinationAuth.Type "none" }}
                     "auth": {}
+                    {{ end }}
                 }
             ]
         }

templates/backbeat/notificationCredentials.json

@@ -0,0 +1,4 @@
+{
+    "username": "{{ .Features.BucketNotifications.DestinationAuth.Username }}",
+    "password": "{{ .Features.BucketNotifications.DestinationAuth.Password }}"
+}

templates/cloudserver/config.json

@@ -89,8 +89,8 @@
             "resource": "destination1",
             "type": "kafka",
             "host": "127.0.0.1",
-            "port": 9093,
-            "topic": "destination-topic",
+            "port": 9094,
+            "topic": "notifications",
             "auth": {}
         }
     ],

templates/global/config.yaml

@@ -7,9 +7,9 @@ features:
     enable_service_user: true
 
   bucket_notifications:
-    enabled: true
-    targetAuth:
-      type: basic
+    enabled: false
+    destinationAuth:
+      type: none
       username: admin
       password: admin123