diff --git a/cmd/config.go b/cmd/config.go index 0541f91..65237ba 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -74,6 +74,7 @@ type FeatureConfig struct { Migration MigrationFeatureConfig `yaml:"migration"` AccessLogging AccessLoggingFeatureConfig `yaml:"access_logging"` S3Frontend S3FrontendFeatureConfig `yaml:"s3_frontend"` + Lifecycle LifecycleFeatureConfig `yaml:"lifecycle"` } type S3FrontendFeatureConfig struct { @@ -240,6 +241,10 @@ type NginxConfig struct { SSLPort uint16 `yaml:"ssl_port"` } +type LifecycleFeatureConfig struct { + Enabled bool `yaml:"enabled"` +} + func DefaultEnvironmentConfig() EnvironmentConfig { return EnvironmentConfig{ Global: GlobalConfig{ @@ -267,6 +272,9 @@ func DefaultEnvironmentConfig() EnvironmentConfig { AccessLogging: AccessLoggingFeatureConfig{ Enabled: false, }, + Lifecycle: LifecycleFeatureConfig{ + Enabled: false, + }, }, Cloudserver: CloudserverConfig{}, S3Metadata: MetadataConfig{ diff --git a/cmd/configure.go b/cmd/configure.go index 24291cd..6c3dcc0 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -49,8 +49,6 @@ func createLogDirectories(envDir string) error { filepath.Join(envDir, "logs", "scuba"), filepath.Join(envDir, "logs", "backbeat"), filepath.Join(envDir, "logs", "migration-tools"), - filepath.Join(envDir, "logs", "clickhouse-shard-1"), - filepath.Join(envDir, "logs", "clickhouse-shard-2"), filepath.Join(envDir, "logs", "fluentbit"), } diff --git a/cmd/util.go b/cmd/util.go index 886ce88..d8cae7f 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -96,6 +96,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string { profiles = append(profiles, "feature-s3-frontend") } + if cfg.Features.Lifecycle.Enabled { + profiles = append(profiles, "feature-lifecycle") + } + return profiles } diff --git a/templates/backbeat/config.json b/templates/backbeat/config.json index 7ee0675..72ab457 100644 --- a/templates/backbeat/config.json +++ b/templates/backbeat/config.json @@ -153,7 +153,7 @@ "lifecycle": { "auth": { "type": "assumeRole", - "roleName": "lifecycle-role", + "roleName": "scality-internal/lifecycle-role", "sts": { "host": "127.0.0.1", "port": 8800, @@ -169,9 +169,21 @@ "bucketTasksTopic": "backbeat-lifecycle-bucket-tasks", "objectTasksTopic": "backbeat-lifecycle-object-tasks", "conductor": { + "auth": { + "type": "none", + "vault": { + "host": "127.0.0.1", + "port": 8500 + } + }, "backlogControl": { "enabled": true }, "cronRule": "*/5 * * * * *", "concurrency": 10, + "bucketSource": "bucketd", + "bucketd": { + "host": "127.0.0.1", + "port": 9000 + }, "probeServer": { "bindAddress": "0.0.0.0", "port": 8552 diff --git a/templates/backbeat/supervisord.conf b/templates/backbeat/supervisord.conf index 11ffee5..efb67a7 100644 --- a/templates/backbeat/supervisord.conf +++ b/templates/backbeat/supervisord.conf @@ -18,6 +18,46 @@ serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock ## CRR ## Lifecycle +{{ if .Features.Lifecycle.Enabled }} +[program:lifecycle-conductor] +command = bash -c "source /conf/env && exec npm run lifecycle_conductor" +numprocs = 1 +process_name = %(program_name)s_%(process_num)s +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true + +[program:lifecycle-bucket-processor] +command = bash -c "source /conf/env && exec npm run lifecycle_bucket_processor" +numprocs = 1 +process_name = %(program_name)s_%(process_num)s +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true + +[program:lifecycle-object-processor] +command = bash -c "source /conf/env && exec npm run lifecycle_object_processor" +numprocs = 1 +process_name = %(program_name)s_%(process_num)s +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true +{{ end }} ## Bucket Notifications {{ if .Features.BucketNotifications.Enabled }} diff --git a/templates/global/docker-compose.yaml b/templates/global/docker-compose.yaml index c2e0e6f..bf5b032 100644 --- a/templates/global/docker-compose.yaml +++ b/templates/global/docker-compose.yaml @@ -95,7 +95,7 @@ services: - ./config/fluentbit/parsers.conf:/fluent-bit/etc/parsers.conf:ro - ./logs/cloudserver:/fluent-bit/log:ro - ./logs/fluentbit:/var/log/fluent-bit:rw - - ./data/fluentbit:/fluent-bit/data:rw + - fluentbit-data:/fluent-bit/data:rw profiles: - feature-access-logging @@ -127,6 +127,7 @@ services: network_mode: host volumes: - ./config/vault/management-creds.json:/conf/management-creds.json:ro + - ./config/backbeat:/conf/backbeat:rw depends_on: vault: condition: service_healthy @@ -177,6 +178,9 @@ services: image: ${BACKBEAT_IMAGE} container_name: workbench-backbeat network_mode: host + depends_on: + setup-vault: + condition: service_completed_successfully environment: SUPERVISORD_CONF: supervisord.conf BACKBEAT_CONFIG_FILE: /conf/config.json @@ -189,6 +193,7 @@ services: profiles: - feature-crr - feature-notifications + - feature-lifecycle redis: image: ${REDIS_IMAGE} @@ -205,6 +210,7 @@ services: - feature-notifications - feature-utapi - feature-migration + - feature-lifecycle zookeeper: build: @@ -217,10 +223,11 @@ services: - ALLOW_ANONYMOUS_LOGIN=yes volumes: - ./config/kafka/zookeeper.properties:/opt/kafka/config/zookeeper.properties:ro - - ./data/zookeeper:/data + - zookeeper-data:/data profiles: - feature-crr - feature-notifications + - feature-lifecycle kafka: build: @@ -231,10 +238,11 @@ services: command: /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties volumes: - ./config/kafka/server.backbeat.properties:/opt/kafka/config/server.properties:ro - - ./data/kafka:/data + - kafka-data:/data profiles: - feature-crr - feature-notifications + - feature-lifecycle setup-kafka: build: @@ -261,6 +269,7 @@ services: profiles: - feature-crr - feature-notifications + - feature-lifecycle kafka-destination: build: @@ -274,7 +283,7 @@ services: volumes: - ./config/kafka/server.destination.properties:/opt/kafka/config/server.properties:ro - ./config/kafka/config.properties:/opt/kafka/config/config.properties:ro - - ./data/kafka-destination:/data + - kafka-destination-data:/data setup-kafka-destination: build: @@ -336,8 +345,8 @@ services: CLICKHOUSE_USER: default CLICKHOUSE_PASSWORD: "" volumes: - - ./data/clickhouse-shard-1:/var/lib/clickhouse - - ./logs/clickhouse-shard-1:/var/log/clickhouse-server:rw + - clickhouse-shard-1-data:/var/lib/clickhouse + - clickhouse-shard-1-logs:/var/log/clickhouse-server - ./config/clickhouse/cluster-config.xml:/etc/clickhouse-server/config.d/cluster.xml:ro - ./config/clickhouse/ports-shard-1.xml:/etc/clickhouse-server/config.d/ports.xml:ro healthcheck: @@ -362,8 +371,8 @@ services: CLICKHOUSE_USER: default CLICKHOUSE_PASSWORD: "" volumes: - - ./data/clickhouse-shard-2:/var/lib/clickhouse - - ./logs/clickhouse-shard-2:/var/log/clickhouse-server:rw + - clickhouse-shard-2-data:/var/lib/clickhouse + - clickhouse-shard-2-logs:/var/log/clickhouse-server - ./config/clickhouse/cluster-config.xml:/etc/clickhouse-server/config.d/cluster.xml:ro - ./config/clickhouse/ports-shard-2.xml:/etc/clickhouse-server/config.d/ports.xml:ro healthcheck: @@ -390,3 +399,13 @@ services: condition: service_healthy profiles: - feature-access-logging + +volumes: + zookeeper-data: + kafka-data: + kafka-destination-data: + fluentbit-data: + clickhouse-shard-1-data: + clickhouse-shard-1-logs: + clickhouse-shard-2-data: + clickhouse-shard-2-logs: diff --git a/templates/global/values.yaml b/templates/global/values.yaml index 017a332..ccf9d85 100644 --- a/templates/global/values.yaml +++ b/templates/global/values.yaml @@ -28,6 +28,9 @@ features: s3_frontend: enabled: false + lifecycle: + enabled: false + cloudserver: image: ghcr.io/scality/cloudserver:9.2.22 diff --git a/templates/kafka/setup.sh b/templates/kafka/setup.sh index 130f49f..1e843f7 100644 --- a/templates/kafka/setup.sh +++ b/templates/kafka/setup.sh @@ -64,6 +64,9 @@ create /queue-populator/raft-id-dispatcher/provisions create /queue-populator/raft-id-dispatcher/provisions/0 create /queue-populator/raft-id-dispatcher/provisions/1 create /queue-populator/raft-id-dispatcher/provisions/2 +create /lifecycle +create /lifecycle/conductor +create /lifecycle/conductor/election quit EOF echo "[setup] Zookeeper paths created." diff --git a/templates/vault/Dockerfile.setup b/templates/vault/Dockerfile.setup index 630688c..49a4a40 100644 --- a/templates/vault/Dockerfile.setup +++ b/templates/vault/Dockerfile.setup @@ -4,7 +4,7 @@ FROM $BASE_IMAGE USER root -RUN apt-get update && apt-get install -y jq +RUN apt-get update && apt-get install -y jq awscli COPY --chmod=755 create-management-account.sh /opt/ diff --git a/templates/vault/config.json b/templates/vault/config.json index 10976ac..dc6b28b 100644 --- a/templates/vault/config.json +++ b/templates/vault/config.json @@ -83,6 +83,38 @@ ] } } + }, + { + "role": { + "roleName": "lifecycle-role", + "trustPolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::000000000000:user/lifecycle" + }, + "Action": "sts:AssumeRole", + "Condition": {} + } + ] + } + }, + "permissionPolicy": { + "policyName": "lifecycle-policy", + "policyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "LifecycleFullAccess", + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": ["*"] + } + ] + } + } } ], "utapi": { diff --git a/templates/vault/create-management-account.sh b/templates/vault/create-management-account.sh index 8197a87..f4206d5 100644 --- a/templates/vault/create-management-account.sh +++ b/templates/vault/create-management-account.sh @@ -53,50 +53,174 @@ if [ $? -ne 0 ]; then exit 1 fi -echo "[setup] Management account and access key setup completed successfully 🎉" - -# === Create acccess key for lifecycle service user === -# echo "[setup] Creating access key for lifecycle service user..." -# LIFECYCLE_CREDS_JSON=$(jq '.extensions.lifecycle.auth.sts' /config/backbeat/config.json) -# LEFECYCLE_ACCESS_KEY=$(echo "$LIFECYCLE_CREDS_JSON" | jq -r '.accessKey') -# LIFECYCLE_SECRET_KEY=$(echo "$LIFECYCLE_CREDS_JSON" | jq -r '.secretKey') - -# # === Generate management access key === -# echo "[setup] Generating management access key..." -# MGMT_CREDS_JSON=$(./node_modules/vaultclient/bin/vaultclient generate-account-access-key --name management --host vault --port 8600) - -# MANAGEMENT_ACCESS_KEY=$(echo "$MGMT_CREDS_JSON" | jq -r '.id') -# MANAGEMENT_SECRET_KEY=$(echo "$MGMT_CREDS_JSON" | jq -r '.value') - -# echo "[setup] Management credentials:" -# echo "MANAGEMENT_ACCESS_KEY=$MANAGEMENT_ACCESS_KEY" -# echo "MANAGEMENT_SECRET_KEY=$MANAGEMENT_SECRET_KEY" -# echo - -# # === Create lifecycle service user === -# echo "[setup] Creating lifecycle service user..." -# SERVICE_CREDS_JSON=$(AWS_ACCESS_KEY_ID="$MANAGEMENT_ACCESS_KEY" \ -# AWS_SECRET_ACCESS_KEY="$MANAGEMENT_SECRET_KEY" \ -# AWS_REGION="$REGION" \ -# ./bin/ensureServiceUser apply lifecycle --iam-endpoint http://vault:8600) - -# SERVICE_ACCESS_KEY=$(echo "$SERVICE_CREDS_JSON" | jq -r '.data.AccessKeyId') -# SERVICE_SECRET_KEY=$(echo "$SERVICE_CREDS_JSON" | jq -r '.data.SecretAccessKey') - -# echo "[setup] Lifecycle service user credentials:" -# echo "SERVICE_ACCESS_KEY=$SERVICE_ACCESS_KEY" -# echo "SERVICE_SECRET_KEY=$SERVICE_SECRET_KEY" -# echo - -# # === Update backbeat-config.json === -# echo "[setup] Updating backbeat-config.json with service user credentials..." -# jq --arg ak "$SERVICE_ACCESS_KEY" --arg sk "$SERVICE_SECRET_KEY" \ -# '.extensions.lifecycle.auth.sts.accessKey = $ak | .extensions.lifecycle.auth.sts.secretKey = $sk' \ -# "$CONFIG_FILE" > /tmp/backbeat-config.updated.json - -# mv /tmp/backbeat-config.updated.json "$CONFIG_FILE" -# echo "[setup] backbeat-config.json successfully updated!" -# echo - -# # === Done === -# echo "[setup] Setup service users completed successfully 🎉" +echo "[setup] Management account and access key setup completed successfully" + +# === Create test data account === +# This account is seeded with the lifecycle role via vault's accountSeeds, +# which allows backbeat to AssumeRole into it for lifecycle operations. +TEST_ACCOUNT_ACCESS_KEY="WBTKACCESSI9O3YKIRQ0" +TEST_ACCOUNT_SECRET_KEY="ICxmNTBbOqijy4rMq/MOP1EPlTMqfsEBLjROcAbN" + +echo "[setup] Creating test data account..." +resp=$(./node_modules/vaultclient/bin/vaultclient \ + create-account \ + --name testaccount \ + --email testaccount@test.com \ + --host 127.0.0.1 \ + --port 8600 2>&1) || { + if echo "$resp" | grep -q "EntityAlreadyExists"; then + echo "[setup] Test data account already exists, skipping creation" + else + echo "[setup] Error creating test data account:" + echo "$resp" + exit 1 + fi +} + +echo "[setup] Generating access key for test data account..." +resp=$(./node_modules/vaultclient/bin/vaultclient \ + generate-account-access-key \ + --name testaccount \ + --accesskey "$TEST_ACCOUNT_ACCESS_KEY" \ + --secretkey "$TEST_ACCOUNT_SECRET_KEY" \ + --host 127.0.0.1 \ + --port 8600 2>&1) || { + if echo "$resp" | grep -q "EntityAlreadyExists"; then + echo "[setup] Test data account access key already exists, skipping" + else + echo "[setup] Error generating access key for test data account:" + echo "$resp" + exit 1 + fi +} + +echo "[setup] Test data account ready (accessKey=$TEST_ACCOUNT_ACCESS_KEY)" + +# === Create lifecycle service user === +BACKBEAT_CONFIG_FILE=/conf/backbeat/config.json +IAM_ENDPOINT=http://127.0.0.1:8600 +export AWS_ACCESS_KEY_ID="$MANAGEMENT_ACCESS_KEY" +export AWS_SECRET_ACCESS_KEY="$MANAGEMENT_SECRET_KEY" +export AWS_DEFAULT_REGION="$REGION" + +if [ -f "$BACKBEAT_CONFIG_FILE" ]; then + echo "[setup] Creating lifecycle service user..." + resp=$(aws iam create-user \ + --user-name lifecycle \ + --endpoint-url "$IAM_ENDPOINT" 2>&1) || { + if echo "$resp" | grep -q "EntityAlreadyExists"; then + echo "[setup] Lifecycle user already exists, skipping creation" + else + echo "[setup] Error creating lifecycle user:" + echo "$resp" + exit 1 + fi + } + + echo "[setup] Generating lifecycle access key..." + LIFECYCLE_CREDS=$(aws iam create-access-key \ + --user-name lifecycle \ + --endpoint-url "$IAM_ENDPOINT" 2>&1) || { + if echo "$LIFECYCLE_CREDS" | grep -q "LimitExceeded\|EntityAlreadyExists"; then + echo "[setup] Lifecycle access key already exists, reading from backbeat config" + SERVICE_ACCESS_KEY=$(jq -r '.extensions.lifecycle.auth.sts.accessKey' "$BACKBEAT_CONFIG_FILE") + SERVICE_SECRET_KEY=$(jq -r '.extensions.lifecycle.auth.sts.secretKey' "$BACKBEAT_CONFIG_FILE") + LIFECYCLE_CREDS="" + else + echo "[setup] Error generating lifecycle access key:" + echo "$LIFECYCLE_CREDS" + exit 1 + fi + } + + if [ -n "$LIFECYCLE_CREDS" ]; then + SERVICE_ACCESS_KEY=$(echo "$LIFECYCLE_CREDS" | jq -r '.AccessKey.AccessKeyId') + SERVICE_SECRET_KEY=$(echo "$LIFECYCLE_CREDS" | jq -r '.AccessKey.SecretAccessKey') + fi + + echo "[setup] Lifecycle user created" + echo "[setup] SERVICE_ACCESS_KEY=$SERVICE_ACCESS_KEY" + + # === Grant lifecycle user permission to assume roles === + echo "[setup] Creating assume-role policy..." + ASSUME_POLICY='{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"sts:AssumeRole","Resource":"*"}}' + resp=$(aws iam create-policy \ + --policy-name lifecycle-assume-role \ + --policy-document "$ASSUME_POLICY" \ + --endpoint-url "$IAM_ENDPOINT" 2>&1) || { + if echo "$resp" | grep -q "EntityAlreadyExists"; then + echo "[setup] Assume-role policy already exists, skipping creation" + else + echo "[setup] Error creating assume-role policy:" + echo "$resp" + exit 1 + fi + } + + aws iam attach-user-policy \ + --policy-arn "arn:aws:iam::000000000000:policy/lifecycle-assume-role" \ + --user-name lifecycle \ + --endpoint-url "$IAM_ENDPOINT" + + if [ $? -ne 0 ]; then + echo "[setup] Error attaching assume-role policy to lifecycle user" + exit 1 + fi + echo "[setup] Assume-role policy attached to lifecycle user" + + # === Create lifecycle role in the internal services account === + # Backbeat hardcodes account 000000000000 for AssumeRole (VAULT-238 workaround), + # and accountSeeds only apply to accounts created via the normal flow, + # so we must create the role explicitly in the internal services account. + echo "[setup] Creating lifecycle role in internal services account..." + TRUST_POLICY="{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::000000000000:user/lifecycle\"},\"Action\":\"sts:AssumeRole\",\"Condition\":{}}]}" + resp=$(aws iam create-role \ + --role-name lifecycle-role \ + --path "/scality-internal/" \ + --assume-role-policy-document "$TRUST_POLICY" \ + --endpoint-url "$IAM_ENDPOINT" 2>&1) || { + if echo "$resp" | grep -q "EntityAlreadyExists"; then + echo "[setup] Lifecycle role already exists, skipping creation" + else + echo "[setup] Error creating lifecycle role:" + echo "$resp" + exit 1 + fi + } + + S3_FULL_ACCESS='{"Version":"2012-10-17","Statement":[{"Sid":"LifecycleFullAccess","Effect":"Allow","Action":["s3:*"],"Resource":["*"]}]}' + resp=$(aws iam create-policy \ + --policy-name lifecycle-s3-access \ + --policy-document "$S3_FULL_ACCESS" \ + --endpoint-url "$IAM_ENDPOINT" 2>&1) || { + if echo "$resp" | grep -q "EntityAlreadyExists"; then + echo "[setup] Lifecycle S3 access policy already exists, skipping creation" + else + echo "[setup] Error creating lifecycle S3 access policy:" + echo "$resp" + exit 1 + fi + } + + aws iam attach-role-policy \ + --role-name lifecycle-role \ + --policy-arn "arn:aws:iam::000000000000:policy/lifecycle-s3-access" \ + --endpoint-url "$IAM_ENDPOINT" + + if [ $? -ne 0 ]; then + echo "[setup] Error setting up lifecycle role" + exit 1 + fi + echo "[setup] Lifecycle role created in internal services account" + + # === Update backbeat config.json with lifecycle credentials === + echo "[setup] Updating backbeat config.json with lifecycle credentials..." + jq --arg ak "$SERVICE_ACCESS_KEY" --arg sk "$SERVICE_SECRET_KEY" \ + '.extensions.lifecycle.auth.sts.accessKey = $ak | .extensions.lifecycle.auth.sts.secretKey = $sk' \ + "$BACKBEAT_CONFIG_FILE" > /tmp/backbeat-config.updated.json + + mv /tmp/backbeat-config.updated.json "$BACKBEAT_CONFIG_FILE" + echo "[setup] Backbeat config.json updated with lifecycle credentials" +fi + +echo "[setup] Setup completed successfully"