WKBCH-24: Add S3 lifecycle expiration support

cmd/config.go

@@ -74,6 +74,7 @@ type FeatureConfig struct {
 	Migration              MigrationFeatureConfig           `yaml:"migration"`
 	AccessLogging          AccessLoggingFeatureConfig       `yaml:"access_logging"`
 	S3Frontend             S3FrontendFeatureConfig          `yaml:"s3_frontend"`
+	Lifecycle              LifecycleFeatureConfig           `yaml:"lifecycle"`
 }
 
 type S3FrontendFeatureConfig struct {
@@ -240,6 +241,10 @@ type NginxConfig struct {
 	SSLPort  uint16 `yaml:"ssl_port"`
 }
 
+type LifecycleFeatureConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
 func DefaultEnvironmentConfig() EnvironmentConfig {
 	return EnvironmentConfig{
 		Global: GlobalConfig{
@@ -267,6 +272,9 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
 			AccessLogging: AccessLoggingFeatureConfig{
 				Enabled: false,
 			},
+			Lifecycle: LifecycleFeatureConfig{
+				Enabled: false,
+			},
 		},
 		Cloudserver: CloudserverConfig{},
 		S3Metadata: MetadataConfig{

cmd/configure.go

@@ -49,8 +49,6 @@ func createLogDirectories(envDir string) error {
 		filepath.Join(envDir, "logs", "scuba"),
 		filepath.Join(envDir, "logs", "backbeat"),
 		filepath.Join(envDir, "logs", "migration-tools"),
-		filepath.Join(envDir, "logs", "clickhouse-shard-1"),
-		filepath.Join(envDir, "logs", "clickhouse-shard-2"),
 		filepath.Join(envDir, "logs", "fluentbit"),
 	}
 

cmd/util.go

@@ -96,6 +96,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string {
 		profiles = append(profiles, "feature-s3-frontend")
 	}
 
+	if cfg.Features.Lifecycle.Enabled {
+		profiles = append(profiles, "feature-lifecycle")
+	}
+
 	return profiles
 }
 

templates/backbeat/config.json

@@ -153,7 +153,7 @@
         "lifecycle": {
             "auth": {
                 "type": "assumeRole",
-                "roleName": "lifecycle-role",
+                "roleName": "scality-internal/lifecycle-role",
                 "sts": {
                     "host": "127.0.0.1",
                     "port": 8800,
@@ -169,9 +169,21 @@
             "bucketTasksTopic": "backbeat-lifecycle-bucket-tasks",
             "objectTasksTopic": "backbeat-lifecycle-object-tasks",
             "conductor": {
+                "auth": {
+                    "type": "none",
+                    "vault": {
+                        "host": "127.0.0.1",
+                        "port": 8500
+                    }
+                },
                 "backlogControl": { "enabled": true },
                 "cronRule": "*/5 * * * * *",
                 "concurrency": 10,
+                "bucketSource": "bucketd",
+                "bucketd": {
+                    "host": "127.0.0.1",
+                    "port": 9000
+                },
                 "probeServer": {
                     "bindAddress": "0.0.0.0",
                     "port": 8552

templates/backbeat/supervisord.conf

@@ -18,6 +18,46 @@ serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
 ## CRR
 
 ## Lifecycle
+{{ if .Features.Lifecycle.Enabled }}
+[program:lifecycle-conductor]
+command = bash -c "source /conf/env && exec npm run lifecycle_conductor"
+numprocs = 1
+process_name = %(program_name)s_%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true
+
+[program:lifecycle-bucket-processor]
+command = bash -c "source /conf/env && exec npm run lifecycle_bucket_processor"
+numprocs = 1
+process_name = %(program_name)s_%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true
+
+[program:lifecycle-object-processor]
+command = bash -c "source /conf/env && exec npm run lifecycle_object_processor"
+numprocs = 1
+process_name = %(program_name)s_%(process_num)s
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true
+{{ end }}
 
 ## Bucket Notifications
 {{ if .Features.BucketNotifications.Enabled }}

templates/global/docker-compose.yaml

@@ -95,7 +95,7 @@ services:
       - ./config/fluentbit/parsers.conf:/fluent-bit/etc/parsers.conf:ro
       - ./logs/cloudserver:/fluent-bit/log:ro
       - ./logs/fluentbit:/var/log/fluent-bit:rw
-      - ./data/fluentbit:/fluent-bit/data:rw
+      - fluentbit-data:/fluent-bit/data:rw
     profiles:
       - feature-access-logging
 
@@ -127,6 +127,7 @@ services:
     network_mode: host
     volumes:
       - ./config/vault/management-creds.json:/conf/management-creds.json:ro
+      - ./config/backbeat:/conf/backbeat:rw
     depends_on:
       vault:
         condition: service_healthy
@@ -177,6 +178,9 @@ services:
     image: ${BACKBEAT_IMAGE}
     container_name: workbench-backbeat
     network_mode: host
+    depends_on:
+      setup-vault:
+        condition: service_completed_successfully
     environment:
       SUPERVISORD_CONF: supervisord.conf
       BACKBEAT_CONFIG_FILE: /conf/config.json
@@ -189,6 +193,7 @@ services:
     profiles:
       - feature-crr
       - feature-notifications
+      - feature-lifecycle
 
   redis:
     image: ${REDIS_IMAGE}
@@ -205,6 +210,7 @@ services:
       - feature-notifications
       - feature-utapi
       - feature-migration
+      - feature-lifecycle
 
   zookeeper:
     build:
@@ -217,10 +223,11 @@ services:
       - ALLOW_ANONYMOUS_LOGIN=yes
     volumes:
       - ./config/kafka/zookeeper.properties:/opt/kafka/config/zookeeper.properties:ro
-      - ./data/zookeeper:/data
+      - zookeeper-data:/data
     profiles:
       - feature-crr
       - feature-notifications
+      - feature-lifecycle
 
   kafka:
     build:
@@ -231,10 +238,11 @@ services:
     command: /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
     volumes:
       - ./config/kafka/server.backbeat.properties:/opt/kafka/config/server.properties:ro
-      - ./data/kafka:/data
+      - kafka-data:/data
     profiles:
       - feature-crr
       - feature-notifications
+      - feature-lifecycle
 
   setup-kafka:
     build:
@@ -261,6 +269,7 @@ services:
     profiles:
       - feature-crr
       - feature-notifications
+      - feature-lifecycle
 
   kafka-destination:
     build:
@@ -274,7 +283,7 @@ services:
     volumes:
       - ./config/kafka/server.destination.properties:/opt/kafka/config/server.properties:ro
       - ./config/kafka/config.properties:/opt/kafka/config/config.properties:ro
-      - ./data/kafka-destination:/data
+      - kafka-destination-data:/data
 
   setup-kafka-destination:
     build:
@@ -336,8 +345,8 @@ services:
       CLICKHOUSE_USER: default
       CLICKHOUSE_PASSWORD: ""
     volumes:
-      - ./data/clickhouse-shard-1:/var/lib/clickhouse
-      - ./logs/clickhouse-shard-1:/var/log/clickhouse-server:rw
+      - clickhouse-shard-1-data:/var/lib/clickhouse
+      - clickhouse-shard-1-logs:/var/log/clickhouse-server
       - ./config/clickhouse/cluster-config.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
       - ./config/clickhouse/ports-shard-1.xml:/etc/clickhouse-server/config.d/ports.xml:ro
     healthcheck:
@@ -362,8 +371,8 @@ services:
       CLICKHOUSE_USER: default
       CLICKHOUSE_PASSWORD: ""
     volumes:
-      - ./data/clickhouse-shard-2:/var/lib/clickhouse
-      - ./logs/clickhouse-shard-2:/var/log/clickhouse-server:rw
+      - clickhouse-shard-2-data:/var/lib/clickhouse
+      - clickhouse-shard-2-logs:/var/log/clickhouse-server
       - ./config/clickhouse/cluster-config.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
       - ./config/clickhouse/ports-shard-2.xml:/etc/clickhouse-server/config.d/ports.xml:ro
     healthcheck:
@@ -390,3 +399,13 @@ services:
         condition: service_healthy
     profiles:
       - feature-access-logging
+
+volumes:
+  zookeeper-data:
+  kafka-data:
+  kafka-destination-data:
+  fluentbit-data:
+  clickhouse-shard-1-data:
+  clickhouse-shard-1-logs:
+  clickhouse-shard-2-data:
+  clickhouse-shard-2-logs:

templates/global/values.yaml

@@ -28,6 +28,9 @@ features:
   s3_frontend:
     enabled: false
 
+  lifecycle:
+    enabled: false
+
 cloudserver:
   image: ghcr.io/scality/cloudserver:9.2.22
 

templates/kafka/setup.sh

@@ -64,6 +64,9 @@ create /queue-populator/raft-id-dispatcher/provisions
 create /queue-populator/raft-id-dispatcher/provisions/0
 create /queue-populator/raft-id-dispatcher/provisions/1
 create /queue-populator/raft-id-dispatcher/provisions/2
+create /lifecycle
+create /lifecycle/conductor
+create /lifecycle/conductor/election
 quit
 EOF
     echo "[setup] Zookeeper paths created."

templates/vault/Dockerfile.setup

@@ -4,7 +4,7 @@ FROM $BASE_IMAGE
 
 USER root
 
-RUN apt-get update && apt-get install -y jq
+RUN apt-get update && apt-get install -y jq awscli
 
 COPY --chmod=755 create-management-account.sh /opt/
 

templates/vault/config.json

@@ -83,6 +83,38 @@
                     ]
                 }
             }
+        },
+        {
+            "role": {
+                "roleName": "lifecycle-role",
+                "trustPolicy": {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Effect": "Allow",
+                            "Principal": {
+                                "AWS": "arn:aws:iam::000000000000:user/lifecycle"
+                            },
+                            "Action": "sts:AssumeRole",
+                            "Condition": {}
+                        }
+                    ]
+                }
+            },
+            "permissionPolicy": {
+                "policyName": "lifecycle-policy",
+                "policyDocument": {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Sid": "LifecycleFullAccess",
+                            "Effect": "Allow",
+                            "Action": ["s3:*"],
+                            "Resource": ["*"]
+                        }
+                    ]
+                }
+            }
         }
     ],
     "utapi": {