diff --git a/addons/kblib/templates/_common.tpl b/addons/kblib/templates/_common.tpl index a4abf61d18..16f567e9f6 100644 --- a/addons/kblib/templates/_common.tpl +++ b/addons/kblib/templates/_common.tpl @@ -31,6 +31,13 @@ policyRules: verbs: - get - list +- apiGroups: + - "dataprotection.kubeblocks.io" + resources: + - backups + verbs: + - get + - list {{- end -}} diff --git a/addons/mongodb/dataprotection/common-scripts.sh b/addons/mongodb/dataprotection/common-scripts.sh index 221f46987f..ed656214d5 100644 --- a/addons/mongodb/dataprotection/common-scripts.sh +++ b/addons/mongodb/dataprotection/common-scripts.sh @@ -170,6 +170,322 @@ function export_pbm_env_vars_for_rs() { export PBM_MONGODB_URI="mongodb://$PBM_AGENT_MONGODB_USERNAME:$PBM_AGENT_MONGODB_PASSWORD@$mongodb_endpoints/?authSource=admin&replSetName=$MONGODB_REPLICA_SET_NAME" } +function write_pbm_storage_config_file() { + local file=$1 + if [ -z "$file" ]; then + echo "ERROR: PBM storage config file path is empty" + exit 1 + fi + mkdir -p "$(dirname "$file")" + cat > "$file" <&2 + exit 1 + fi + echo "${pod_name}.${component_name}-headless.${namespace}.svc.${cluster_domain}" +} + +function syncerctl_cmd() { + local host + host=$(target_syncer_host) + local port="${SYNCER_SERVICE_PORT:-3601}" + syncerctl --host "$host" --port "$port" "$@" +} + +function expected_restore_shard_names_json() { + local shardsvr_names="${MONGODB_SHARD_REPLICA_SET_NAME_LIST:-}" + if [ -z "$shardsvr_names" ]; then + echo "[]" + return + fi + + local -a shardsvr_array + IFS="." read -r -a shardsvr_array <<< "$shardsvr_names" + local shardsvr_count=${#shardsvr_array[@]} + local json="[" + local i + for i in "${!shardsvr_array[@]}"; do + local shard_name + if [ "$shardsvr_count" -gt 1 ]; then + shard_name="$CLUSTER_NAME-${shardsvr_array[i]%%@*}" + else + shard_name="${shardsvr_array[i]%%,*}" + fi + if [ -z "$shard_name" ]; then + continue + fi + local escaped="$shard_name" + escaped=${escaped//\\/\\\\} + escaped=${escaped//\"/\\\"} + if [ "$json" != "[" ]; then + json="$json," + fi + json="$json\"$escaped\"" + done + echo "$json]" +} + +function wait_for_mongos_router_ready() { + if [ -z "${MONGOS_INTERNAL_HOST:-}" ] || [ -z "${MONGOS_INTERNAL_PORT:-}" ]; then + echo "ERROR: Cannot wait for mongos router, host=${MONGOS_INTERNAL_HOST:-} port=${MONGOS_INTERNAL_PORT:-}" + exit 1 + fi + + local client="mongosh" + if ! command -v mongosh >/dev/null 2>&1; then + client="mongo" + fi + + local expected_shards + expected_shards=$(expected_restore_shard_names_json) + local max_retries=${MONGOS_ROUTE_WAIT_MAX_RETRIES:-90} + local retry_interval=${MONGOS_ROUTE_WAIT_INTERVAL_SECONDS:-2} + local settle_seconds=${MONGOS_ROUTE_SETTLE_SECONDS:-20} + local eval_timeout=${MONGOS_ROUTE_EVAL_TIMEOUT_SECONDS:-15} + local attempt=1 + local script + script=$(cat < 0) { + print('missing shards: ' + missing.join(',')); + quit(5); +} +var configShardCursorCount = db.getSiblingDB('config').shards.find({ state: 1 }).limit(Math.max(expected.length, 1)).itcount(); +if (configShardCursorCount < Math.max(expected.length, 1)) { + print('config.shards cursor saw ' + configShardCursorCount + ' active shards, expected at least ' + Math.max(expected.length, 1)); + quit(6); +} +print('router ready: ' + JSON.stringify(found)); +EOF +) + + while [ "$attempt" -le "$max_retries" ]; do + local result + local timeout_cmd=() + if command -v timeout >/dev/null 2>&1; then + timeout_cmd=(timeout -k 2s "${eval_timeout}s") + fi + set +e + result=$("${timeout_cmd[@]}" "$client" --host "$MONGOS_INTERNAL_HOST" --port "$MONGOS_INTERNAL_PORT" -u "$MONGODB_USER" -p "$MONGODB_PASSWORD" --authenticationDatabase admin --quiet --eval "$script" 2>&1) + local exit_code + exit_code=$? + set -e + if [ "$exit_code" -eq 0 ]; then + echo "INFO: Mongos router is ready: $result" + if [ "$settle_seconds" -gt 0 ]; then + echo "INFO: Waiting ${settle_seconds}s for mongos router cache to settle." + sleep "$settle_seconds" + fi + return 0 + fi + echo "INFO: Waiting for mongos router to be ready... (attempt $attempt/$max_retries): $result" + attempt=$((attempt+1)) + sleep "$retry_interval" + done + + echo "ERROR: Mongos router failed to become ready after $max_retries attempts." + exit 1 +} + +function configure_syncer_backup() { + local cnf_file="${MOUNT_DIR:-/tmp}/tmp/pbm_syncer_storage.yaml" + write_pbm_storage_config_file "$cnf_file" + echo "INFO: Configuring PBM storage through syncer on $(target_syncer_host)..." + syncerctl_cmd backup configure --file "$cnf_file" +} + +function ensure_restore_coord_storage_config() { + local cnf_file="${MOUNT_DIR:-/tmp}/tmp/pbm_restore_syncer_storage.yaml" + local coord_cm="${CLUSTER_NAME}-restore-coord" + local namespace="${CLUSTER_NAMESPACE:-${KB_NAMESPACE:-${POD_NAMESPACE:-}}}" + if [ -z "$CLUSTER_NAME" ] || [ -z "$namespace" ]; then + echo "ERROR: Cannot prepare restore coord ConfigMap, cluster=$CLUSTER_NAME namespace=$namespace" + exit 1 + fi + write_pbm_storage_config_file "$cnf_file" + if ! kubectl get configmap "$coord_cm" -n "$namespace" >/dev/null 2>&1; then + kubectl create configmap "$coord_cm" -n "$namespace" >/dev/null 2>&1 || kubectl get configmap "$coord_cm" -n "$namespace" >/dev/null + fi + kubectl label configmap "$coord_cm" -n "$namespace" app.kubernetes.io/instance="$CLUSTER_NAME" --overwrite >/dev/null + local cfg_json + cfg_json=$(jq -Rs . < "$cnf_file") + kubectl patch configmap "$coord_cm" -n "$namespace" --type=merge -p "{\"data\":{\"pbm-storage-config\":$cfg_json}}" >/dev/null + echo "INFO: Restore coord storage config prepared in $namespace/$coord_cm." +} + +function wait_for_syncer_backup_completion() { + local backup_name=$1 + local max_retries=${SYNCER_PBM_WAIT_MAX_RETRIES:-720} + local retry_interval=${SYNCER_PBM_WAIT_INTERVAL_SECONDS:-5} + local attempt=0 + describe_result="" + while true; do + describe_result=$(syncerctl_cmd backup status --op-id "$backup_name") + local found + local status + found=$(echo "$describe_result" | jq -r '.found // false') + status=$(echo "$describe_result" | jq -r '.status // empty') + echo "INFO: Backup $backup_name status: found=$found status=$status" + if [ "$found" = "true" ] && [ "$status" = "done" ]; then + return 0 + fi + if [ "$status" = "error" ] || [ "$status" = "failed" ]; then + echo "ERROR: Backup $backup_name failed: $(echo "$describe_result" | jq -r '.error // empty')" + exit 1 + fi + attempt=$((attempt+1)) + if [ $attempt -gt $max_retries ]; then + echo "ERROR: Backup $backup_name did not complete after $max_retries retries" + exit 1 + fi + sleep "$retry_interval" + done +} + +function wait_for_syncer_restore_completion() { + local coord_cm="${CLUSTER_NAME}-restore-coord" + local namespace="${CLUSTER_NAMESPACE:-${KB_NAMESPACE:-${POD_NAMESPACE:-}}}" + local max_retries=${SYNCER_RESTORE_WAIT_MAX_RETRIES:-7200} + local retry_interval=${SYNCER_RESTORE_WAIT_INTERVAL_SECONDS:-1} + local attempt=0 + local last_phase="" + local last_op_id="" + while true; do + set +e + local cm_json + cm_json=$(kubectl get configmap "$coord_cm" -n "$namespace" -o json 2>/dev/null) + local get_exit=$? + set -e + if [ $get_exit -ne 0 ]; then + if [ "$last_phase" = "done" ] || [ "$last_phase" = "finalizing" ]; then + echo "INFO: Restore coord ConfigMap was removed after phase=$last_phase; treating restore as completed." + return 0 + fi + if [ -n "$last_op_id" ]; then + set +e + local restore_status + restore_status=$(syncerctl_cmd restore status --op-id "$last_op_id" 2>/dev/null) + local status_exit=$? + set -e + if [ $status_exit -eq 0 ]; then + local status + status=$(echo "$restore_status" | jq -r '.status // empty') + if [ "$status" = "done" ]; then + echo "INFO: Restore $last_op_id completed after coord cleanup." + return 0 + fi + if [ "$status" = "failed" ] || [ "$status" = "error" ]; then + echo "ERROR: Syncer restore failed after coord cleanup: $(echo "$restore_status" | jq -r '.error // empty')" + exit 1 + fi + fi + fi + echo "INFO: Waiting for restore coord ConfigMap $namespace/$coord_cm..." + else + local phase + local op_id + local err_msg + local state_json + state_json=$(echo "$cm_json" | jq -r '.data.state // empty') + if [ -n "$state_json" ]; then + phase=$(echo "$state_json" | jq -r '.phase // empty') + op_id=$(echo "$state_json" | jq -r '.op_id // empty') + err_msg=$(echo "$state_json" | jq -r '.error // empty') + else + phase="" + op_id="" + err_msg="" + fi + if [ -z "$phase" ]; then + phase=$(echo "$cm_json" | jq -r '.metadata.annotations["restore.syncer/phase"] // empty') + fi + if [ -z "$op_id" ]; then + op_id=$(echo "$cm_json" | jq -r '.metadata.annotations["restore.syncer/op-id"] // empty') + fi + if [ -z "$err_msg" ]; then + err_msg=$(echo "$cm_json" | jq -r '.metadata.annotations["restore.syncer/error"] // empty') + fi + if [ -n "$op_id" ]; then + last_op_id="$op_id" + fi + if [ -n "$phase" ] && [ "$phase" != "$last_phase" ]; then + echo "INFO: Restore coord phase=$phase op_id=$op_id" + last_phase="$phase" + fi + if [ "$phase" = "done" ]; then + return 0 + fi + if [ "$phase" = "failed" ]; then + echo "ERROR: Syncer restore failed: $err_msg" + exit 1 + fi + fi + attempt=$((attempt+1)) + if [ $attempt -gt $max_retries ]; then + echo "ERROR: Restore did not complete after $max_retries retries" + exit 1 + fi + sleep "$retry_interval" + done +} + function sync_pbm_storage_config() { echo "INFO: Checking if PBM storage config exists" pbm_config_exists=true diff --git a/addons/mongodb/dataprotection/rs-pbm-full-backup.sh b/addons/mongodb/dataprotection/rs-pbm-full-backup.sh index 346194415e..02a48b20ce 100644 --- a/addons/mongodb/dataprotection/rs-pbm-full-backup.sh +++ b/addons/mongodb/dataprotection/rs-pbm-full-backup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e set -o pipefail -export PATH="$PATH:$DP_DATASAFED_BIN_PATH" +export PATH="$PATH:$DP_DATASAFED_BIN_PATH:${MOUNT_DIR}/tmp/bin" export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" trap handle_exit EXIT @@ -16,28 +16,49 @@ trap handle_backup_exit EXIT wait_for_other_operations -sync_pbm_storage_config +configure_syncer_backup wait_for_other_operations -echo "INFO: Starting $PBM_BACKUP_TYPE backup for MongoDB..." -backup_result=$(pbm backup --type=$PBM_BACKUP_TYPE --mongodb-uri "$PBM_MONGODB_URI" --compression=$PBM_COMPRESSION --wait -o json) -backup_name=$(echo "$backup_result" | jq -r '.name') +echo "INFO: Starting $PBM_BACKUP_TYPE backup for MongoDB through syncer..." +backup_result=$(syncerctl_cmd backup start --type="$PBM_BACKUP_TYPE" --compression="$PBM_COMPRESSION") +backup_name=$(echo "$backup_result" | jq -r '.op_id // empty') +if [ -z "$backup_name" ] || [ "$backup_name" = "null" ]; then + echo "ERROR: syncer backup start did not return op_id: $backup_result" + exit 1 +fi extras=$(buildJsonString "" "backup_name" "$backup_name") extras=$(buildJsonString $extras "backup_type" "$PBM_BACKUP_TYPE") echo "INFO: Backup name: $backup_name" -wait_for_backup_completion +wait_for_syncer_backup_completion "$backup_name" -echo "INFO: Backup description result:" -echo "$(echo $describe_result | jq)" -last_write_time=$(echo "$describe_result" | jq -r '.last_write_time') +echo "INFO: Backup status result:" +echo "$(echo "$describe_result" | jq)" + +rs_name=$(echo "$describe_result" | jq -r '[.replsets[]? | .name] | .[0] // empty') +if [ -z "$rs_name" ] || [ "$rs_name" = "null" ]; then + rs_name="$MONGODB_REPLICA_SET_NAME" +fi +extras=$(buildJsonString $extras "replicaset" "$rs_name") + +last_write_unix=$(echo "$describe_result" | jq -r '.last_write_ts.t // .last_write_ts.T // .last_write_ts // empty') +if [[ "$last_write_unix" =~ ^[0-9]+$ ]]; then + last_write_time=$(date -u -d "@${last_write_unix}" +"%Y-%m-%dT%H:%M:%SZ") + end_time=$(date -u -d "@$((last_write_unix + 2))" +"%Y-%m-%dT%H:%M:%SZ") +else + last_write_time="" + end_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +fi extras=$(buildJsonString $extras "last_write_time" "$last_write_time") -start_time=$(echo "$describe_result" | jq -r '.name') -end_unix_time=$(echo "$describe_result" | jq -r '.last_write_ts') -# used for pitr -end_time=$(date -u -d "@$((end_unix_time + 2))" +"%Y-%m-%dT%H:%M:%SZ") -total_size=$(echo "$describe_result" | jq -r '.size') + +start_unix=$(echo "$describe_result" | jq -r '.start_ts // empty') +if [[ "$start_unix" =~ ^[0-9]+$ ]]; then + start_time=$(date -u -d "@${start_unix}" +"%Y-%m-%dT%H:%M:%SZ") +else + start_time="$backup_name" +fi +total_size=$(echo "$describe_result" | jq -r '.size // 0') DP_save_backup_status_info "$total_size" "$start_time" "$end_time" "" "{$extras}" print_pbm_logs_by_event "backup" diff --git a/addons/mongodb/dataprotection/rs-pbm-full-restore.sh b/addons/mongodb/dataprotection/rs-pbm-full-restore.sh index 5d641ec385..26ee93fee2 100644 --- a/addons/mongodb/dataprotection/rs-pbm-full-restore.sh +++ b/addons/mongodb/dataprotection/rs-pbm-full-restore.sh @@ -14,31 +14,40 @@ trap handle_restore_exit EXIT wait_for_other_operations -sync_pbm_storage_config - -sync_pbm_config_from_storage +ensure_restore_coord_storage_config extras=$(cat /dp_downward/status_extras) -backup_name=$(echo "$extras" | jq -r '.[0].backup_name') -backup_type=$(echo "$extras" | jq -r '.[0].backup_type') +backup_name=$(echo "$extras" | jq -r '.[0].backup_name // empty') +backup_type=$(echo "$extras" | jq -r '.[0].backup_type // empty') if [ -z "$backup_type" ] || [ -z "$backup_name" ]; then echo "ERROR: Backup type or backup name is empty, skip restore." exit 1 fi -get_describe_backup_info +rs_name=$(echo "$extras" | jq -r '.[0].replicaset // empty') +if [ -z "$rs_name" ] || [ "$rs_name" = "null" ]; then + echo "INFO: Backup extras do not contain replset mapping metadata, falling back to pbm describe-backup." + sync_pbm_storage_config + sync_pbm_config_from_storage + get_describe_backup_info + rs_name=$(echo "$describe_result" | jq -r '[.replsets[]? | .name] | .[0] // empty') +fi +if [ -z "$rs_name" ] || [ "$rs_name" = "null" ]; then + echo "ERROR: Missing source replica set metadata for restore mapping." + exit 1 +fi -rs_name=$(echo "$describe_result" | jq -r '.replsets[0].name') mappings="$MONGODB_REPLICA_SET_NAME=$rs_name" echo "INFO: Replica set mappings: $mappings" -process_restore_start_signal - -wait_for_other_operations - -restore_name=$(pbm restore $backup_name --mongodb-uri "$PBM_MONGODB_URI" --replset-remapping "$mappings" -o json | jq -r '.name') +echo "INFO: Starting syncer physical restore..." +if ! restore_result=$(syncerctl_cmd restore start --backup-name "$backup_name" --type physical --replset-remapping "$mappings" 2>&1); then + echo "ERROR: Syncer restore start failed: $restore_result" + exit 1 +fi +echo "INFO: Syncer restore start result: $restore_result" -wait_for_restoring +wait_for_syncer_restore_completion -process_restore_end_signal +echo "INFO: Restore completed." diff --git a/addons/mongodb/dataprotection/rs-pbm-pitr-backup.sh b/addons/mongodb/dataprotection/rs-pbm-pitr-backup.sh index 82048ad019..419fe058b3 100644 --- a/addons/mongodb/dataprotection/rs-pbm-pitr-backup.sh +++ b/addons/mongodb/dataprotection/rs-pbm-pitr-backup.sh @@ -1,60 +1,72 @@ #!/bin/bash -# When doing point-in-time recovery for deployments with sharded collections, PBM only writes data to existing ones and doesn’t support creating new collections. -# Therefore, whenever you create a new sharded collection, make a new backup for it to be included there. Ref: https://docs.percona.com/percona-backup-mongodb/usage/pitr-physical.html#post-restore-steps +# For PITR with PBM physical backups, create a fresh base backup whenever new collections must be included by restore. +# Ref: https://docs.percona.com/percona-backup-mongodb/usage/pitr-physical.html#post-restore-steps set -e set -o pipefail -export PATH="$PATH:$DP_DATASAFED_BIN_PATH" +export PATH="$PATH:$DP_DATASAFED_BIN_PATH:${MOUNT_DIR}/tmp/bin" export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" trap handle_exit EXIT function enable_pitr() { - local current_pitr_conf=$(pbm config --mongodb-uri "$PBM_MONGODB_URI" -o json | jq -r '.pitr') - local current_pitr_enabled=$(echo $current_pitr_conf | jq -r '.enabled') - local current_oplog_span_min=$(echo $current_pitr_conf | jq -r '.oplogSpanMin') - local current_pitr_compression=$(echo $current_pitr_conf | jq -r '.compression') + local current_pitr_conf + current_pitr_conf=$(syncerctl_cmd pitr status) + local current_pitr_enabled + local current_oplog_span_min + local current_pitr_compression + current_pitr_enabled=$(echo "$current_pitr_conf" | jq -r '.enabled // false') + current_oplog_span_min=$(echo "$current_pitr_conf" | jq -r '.oplog_span_min // empty') + current_pitr_compression=$(echo "$current_pitr_conf" | jq -r '.compression // empty') if [ "$current_pitr_enabled" != "true" ] || [ "$current_oplog_span_min" != "$PBM_OPLOG_SPAN_MIN_MINUTES" ] || [ "$current_pitr_compression" != "$PBM_COMPRESSION" ]; then - echo "INFO: Pitr config is not equal to the current config, updating..." + echo "INFO: PITR config is not equal to the desired config, updating through syncer..." wait_for_other_operations "backup" - - pbm config --mongodb-uri "$PBM_MONGODB_URI" --set pitr.enabled=true --set pitr.oplogSpanMin=$PBM_OPLOG_SPAN_MIN_MINUTES --set pitr.compression=$PBM_COMPRESSION - echo "INFO: Pitr config updated." + syncerctl_cmd pitr enable --oplog-span-min "$PBM_OPLOG_SPAN_MIN_MINUTES" --compression "$PBM_COMPRESSION" + echo "INFO: PITR config updated." fi } function disable_pitr() { - echo "INFO: Disabling Pitr..." - pbm config --set pitr.enabled=false --mongodb-uri "$PBM_MONGODB_URI" - echo "INFO: Pitr disabled." + echo "INFO: Disabling PITR through syncer..." + syncerctl_cmd pitr disable + echo "INFO: PITR disabled." } function upload_continuous_backup_info() { - local status_result=$(pbm status --mongodb-uri "$PBM_MONGODB_URI" -o json) + local status_result + status_result=$(syncerctl_cmd pitr chunks) echo "INFO: Uploading continuous backup info..." - local pitr_chunks_result=$(echo "$status_result" | jq -r '.backups.pitrChunks') echo "INFO: Continuous backup result:" - echo "$(echo $pitr_chunks_result | jq)" - local pitr_chunks_arr=$(echo "$pitr_chunks_result" | jq -r '.pitrChunks') + echo "$(echo "$status_result" | jq)" + local pitr_chunks_arr + pitr_chunks_arr=$(echo "$status_result" | jq -r '.pitrChunks') if [ -z "$pitr_chunks_arr" ] || [ "$pitr_chunks_arr" = "null" ] || [ "$pitr_chunks_arr" = "[]" ]; then echo "INFO: No oplog found." return fi - local filtered_sorted_chunks=$(echo "$pitr_chunks_arr" | jq 'map(select(.noBaseSnapshot != true)) | sort_by(.range.end)') + local filtered_sorted_chunks + filtered_sorted_chunks=$(echo "$pitr_chunks_arr" | jq 'map(select(.noBaseSnapshot != true)) | sort_by(.range.end)') if [ -z "$filtered_sorted_chunks" ] || [ "$filtered_sorted_chunks" = "null" ] || [ "$filtered_sorted_chunks" = "[]" ]; then echo "INFO: No oplog found." return fi - local last_chunk=$(echo "$filtered_sorted_chunks" | jq -r '.[-1].range') - local start_unix_time=$(echo "$last_chunk" | jq -r '.start') - local end_unix_time=$(echo "$last_chunk" | jq -r '.end') - local start_time=$(date -u -d "@${start_unix_time}" +"%Y-%m-%dT%H:%M:%SZ") - local end_time=$(date -u -d "@${end_unix_time}" +"%Y-%m-%dT%H:%M:%SZ") - local total_size=$(echo "$pitr_chunks_result" | jq -r '.size') + local last_chunk + last_chunk=$(echo "$filtered_sorted_chunks" | jq -r '.[-1].range') + local start_unix_time + local end_unix_time + start_unix_time=$(echo "$last_chunk" | jq -r '.start') + end_unix_time=$(echo "$last_chunk" | jq -r '.end') + local start_time + local end_time + start_time=$(date -u -d "@${start_unix_time}" +"%Y-%m-%dT%H:%M:%SZ") + end_time=$(date -u -d "@${end_unix_time}" +"%Y-%m-%dT%H:%M:%SZ") + local total_size + total_size=$(echo "$status_result" | jq -r '.size // 0') local backup_type="continuous" - local extras=$(buildJsonString "" "backup_type" "$backup_type") + local extras + extras=$(buildJsonString "" "backup_type" "$backup_type") extras=$(buildJsonString $extras "replicaset" "$MONGODB_REPLICA_SET_NAME") DP_save_backup_status_info "$total_size" "$start_time" "$end_time" "" "{$extras}" @@ -64,7 +76,6 @@ function upload_continuous_backup_info() { function purge_pitr_chunks() { current_time=$(date +%s) - # Initialize last_global_purge_time if not set if [ -z "$last_global_purge_time" ]; then last_global_purge_time=0 fi @@ -75,34 +86,32 @@ function purge_pitr_chunks() { return fi - # Update last purge time last_global_purge_time=$current_time - echo "INFO: Purging PBM chunks..." - wait_for_other_operations "backup" - - pbm config --force-resync --mongodb-uri "$PBM_MONGODB_URI" - - # resync wait flag might don't work + echo "INFO: Purging PBM chunks through syncer..." wait_for_other_operations "backup" - local pitr_chunks_result=$(pbm status --mongodb-uri "$PBM_MONGODB_URI" -o json | jq -r '.backups.pitrChunks') - local pitr_chunks_arr=$(echo "$pitr_chunks_result" | jq -r '.pitrChunks') + local status_result + local pitr_chunks_arr + status_result=$(syncerctl_cmd pitr chunks) + pitr_chunks_arr=$(echo "$status_result" | jq -r '.pitrChunks') if [ -z "$pitr_chunks_arr" ] || [ "$pitr_chunks_arr" = "null" ] || [ "$pitr_chunks_arr" = "[]" ]; then - echo "INFO: No no base snapshot chunks found." + echo "INFO: No no-base-snapshot chunks found." return fi - local filtered_sorted_chunks=$(echo "$pitr_chunks_arr" | jq -e 'map(select(.noBaseSnapshot == true)) | sort_by(.range.end)') + local filtered_sorted_chunks + filtered_sorted_chunks=$(echo "$pitr_chunks_arr" | jq -e 'map(select(.noBaseSnapshot == true)) | sort_by(.range.end)') if [ -z "$filtered_sorted_chunks" ] || [ "$filtered_sorted_chunks" = "null" ] || [ "$filtered_sorted_chunks" = "[]" ]; then - echo "INFO: No no base snapshot chunks." + echo "INFO: No no-base-snapshot chunks." return fi - echo "INFO: No base snapshot chunks:" - echo "$(echo $filtered_sorted_chunks | jq)" - local first_chunk_end=$(echo "$filtered_sorted_chunks" | jq -r '.[0].range.end') + echo "INFO: No-base-snapshot chunks:" + echo "$(echo "$filtered_sorted_chunks" | jq)" + local first_chunk_end + first_chunk_end=$(echo "$filtered_sorted_chunks" | jq -r '.[0].range.end') purge_time=$(date -u -d "@${first_chunk_end}" +"%Y-%m-%dT%H:%M:%S") - pbm cleanup --older-than $purge_time --mongodb-uri "$PBM_MONGODB_URI" --wait --yes - echo "INFO: No base snapshot chunks cleaned up." + syncerctl_cmd pitr cleanup --older-than "$purge_time" + echo "INFO: No-base-snapshot chunks cleanup requested." } export_pbm_env_vars_for_rs @@ -113,15 +122,13 @@ export_logs_start_time_env trap handle_pitr_exit EXIT -sync_pbm_storage_config - -sync_pbm_config_from_storage +# Apply storage once. Re-applying the storage file in the loop clears PBM PITR settings +# and restarts slicing before chunks can mature. +configure_syncer_backup while true; do wait_for_other_operations "backup" - sync_pbm_storage_config - enable_pitr purge_pitr_chunks @@ -133,4 +140,4 @@ while true; do export_logs_start_time_env sleep 30 -done \ No newline at end of file +done diff --git a/addons/mongodb/dataprotection/rs-pbm-pitr-restore.sh b/addons/mongodb/dataprotection/rs-pbm-pitr-restore.sh index f9d1f26353..c2fe304934 100644 --- a/addons/mongodb/dataprotection/rs-pbm-pitr-restore.sh +++ b/addons/mongodb/dataprotection/rs-pbm-pitr-restore.sh @@ -14,28 +14,28 @@ trap handle_restore_exit EXIT wait_for_other_operations -sync_pbm_storage_config - -sync_pbm_config_from_storage - -process_restore_start_signal +ensure_restore_coord_storage_config extras=$(cat /dp_downward/status_extras) -rs_name=$(echo "$extras" | jq -r '.[0].replicaset') +rs_name=$(echo "$extras" | jq -r '.[0].replicaset // empty') +if [ -z "$rs_name" ] || [ "$rs_name" = "null" ]; then + echo "ERROR: Missing source replica set metadata for PITR restore mapping." + exit 1 +fi + mappings="$MONGODB_REPLICA_SET_NAME=$rs_name" echo "INFO: Replica set mappings: $mappings" recovery_target_time=$(date -d "@${DP_RESTORE_TIMESTAMP}" +"%Y-%m-%dT%H:%M:%S") echo "INFO: Recovery target time: $recovery_target_time" -echo "INFO: Starting restore..." - -wait_for_other_operations - -restore_name=$(pbm restore --time="$recovery_target_time" --mongodb-uri "$PBM_MONGODB_URI" --replset-remapping "$mappings" -o json | jq -r '.name') - -wait_for_restoring +echo "INFO: Starting syncer PITR restore..." +if ! restore_result=$(syncerctl_cmd restore start --pitr-target "$recovery_target_time" --type physical --replset-remapping "$mappings" 2>&1); then + echo "ERROR: Syncer restore start failed: $restore_result" + exit 1 +fi +echo "INFO: Syncer restore start result: $restore_result" -process_restore_end_signal +wait_for_syncer_restore_completion echo "INFO: Restore completed." diff --git a/addons/mongodb/dataprotection/rs-pbm-restore-prepare.sh b/addons/mongodb/dataprotection/rs-pbm-restore-prepare.sh deleted file mode 100644 index d99caf0326..0000000000 --- a/addons/mongodb/dataprotection/rs-pbm-restore-prepare.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -mkdir -p ${MOUNT_DIR}/tmp - -cd ${MOUNT_DIR}/tmp && touch mongodb_pbm.backup \ No newline at end of file diff --git a/addons/mongodb/dataprotection/shard-pbm-full-backup.sh b/addons/mongodb/dataprotection/shard-pbm-full-backup.sh index 0f0aa7f4c2..45a7bc4d15 100644 --- a/addons/mongodb/dataprotection/shard-pbm-full-backup.sh +++ b/addons/mongodb/dataprotection/shard-pbm-full-backup.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e set -o pipefail -export PATH="$PATH:$DP_DATASAFED_BIN_PATH" +export PATH="$PATH:$DP_DATASAFED_BIN_PATH:${MOUNT_DIR}/tmp/bin" export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" trap handle_exit EXIT @@ -16,28 +16,52 @@ trap handle_backup_exit EXIT wait_for_other_operations -sync_pbm_storage_config +configure_syncer_backup wait_for_other_operations -echo "INFO: Starting $PBM_BACKUP_TYPE backup for MongoDB..." -backup_result=$(pbm backup --type=$PBM_BACKUP_TYPE --mongodb-uri "$PBM_MONGODB_URI" --compression=$PBM_COMPRESSION --wait -o json) -backup_name=$(echo "$backup_result" | jq -r '.name') +echo "INFO: Starting $PBM_BACKUP_TYPE backup for MongoDB through syncer..." +backup_result=$(syncerctl_cmd backup start --type="$PBM_BACKUP_TYPE" --compression="$PBM_COMPRESSION") +backup_name=$(echo "$backup_result" | jq -r '.op_id // empty') +if [ -z "$backup_name" ] || [ "$backup_name" = "null" ]; then + echo "ERROR: syncer backup start did not return op_id: $backup_result" + exit 1 +fi extras=$(buildJsonString "" "backup_name" "$backup_name") extras=$(buildJsonString $extras "backup_type" "$PBM_BACKUP_TYPE") echo "INFO: Backup name: $backup_name" -wait_for_backup_completion +wait_for_syncer_backup_completion "$backup_name" -echo "INFO: Backup description result:" -echo "$(echo $describe_result | jq)" -last_write_time=$(echo "$describe_result" | jq -r '.last_write_time') +echo "INFO: Backup status result:" +echo "$(echo "$describe_result" | jq)" + +last_write_unix=$(echo "$describe_result" | jq -r '.last_write_ts.t // .last_write_ts.T // .last_write_ts // empty') +if [[ "$last_write_unix" =~ ^[0-9]+$ ]]; then + last_write_time=$(date -u -d "@${last_write_unix}" +"%Y-%m-%dT%H:%M:%SZ") + end_time=$(date -u -d "@$((last_write_unix + 2))" +"%Y-%m-%dT%H:%M:%SZ") +else + last_write_time="" + end_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +fi extras=$(buildJsonString $extras "last_write_time" "$last_write_time") -start_time=$(echo "$describe_result" | jq -r '.name') -end_unix_time=$(echo "$describe_result" | jq -r '.last_write_ts') -# used for pitr -end_time=$(date -u -d "@$((end_unix_time + 2))" +"%Y-%m-%dT%H:%M:%SZ") -total_size=$(echo "$describe_result" | jq -r '.size') + +configsvr_name=$(echo "$describe_result" | jq -r '[.replsets[]? | select((.configsvr // false) == true or (.iscs // false) == true) | .name] | .[0] // empty') +shardsvr_names=$(echo "$describe_result" | jq -r '[.replsets[]? | select(((.configsvr // false) != true) and ((.iscs // false) != true)) | .name] | join(",")') +if [ -n "$configsvr_name" ]; then + extras=$(buildJsonString $extras "configsvr" "$configsvr_name") +fi +if [ -n "$shardsvr_names" ]; then + extras=$(buildJsonString $extras "shardsvr" "$shardsvr_names") +fi + +start_unix=$(echo "$describe_result" | jq -r '.start_ts // empty') +if [[ "$start_unix" =~ ^[0-9]+$ ]]; then + start_time=$(date -u -d "@${start_unix}" +"%Y-%m-%dT%H:%M:%SZ") +else + start_time="$backup_name" +fi +total_size=$(echo "$describe_result" | jq -r '.size // 0') DP_save_backup_status_info "$total_size" "$start_time" "$end_time" "" "{$extras}" print_pbm_logs_by_event "backup" diff --git a/addons/mongodb/dataprotection/shard-pbm-full-restore.sh b/addons/mongodb/dataprotection/shard-pbm-full-restore.sh index 375b8587e7..bfc66c8521 100644 --- a/addons/mongodb/dataprotection/shard-pbm-full-restore.sh +++ b/addons/mongodb/dataprotection/shard-pbm-full-restore.sh @@ -14,25 +14,35 @@ trap handle_restore_exit EXIT wait_for_other_operations -sync_pbm_storage_config - -sync_pbm_config_from_storage +ensure_restore_coord_storage_config extras=$(cat /dp_downward/status_extras) -backup_name=$(echo "$extras" | jq -r '.[0].backup_name') -backup_type=$(echo "$extras" | jq -r '.[0].backup_type') +backup_name=$(echo "$extras" | jq -r '.[0].backup_name // empty') +backup_type=$(echo "$extras" | jq -r '.[0].backup_type // empty') if [ -z "$backup_type" ] || [ -z "$backup_name" ]; then echo "ERROR: Backup type or backup name is empty, skip restore." exit 1 fi -get_describe_backup_info +configsvr_name=$(echo "$extras" | jq -r '.[0].configsvr // empty') +shardsvr_names=$(echo "$extras" | jq -r '.[0].shardsvr // empty') +if [ -z "$configsvr_name" ] || [ -z "$shardsvr_names" ]; then + echo "INFO: Backup extras do not contain replset mapping metadata, falling back to pbm describe-backup." + sync_pbm_storage_config + sync_pbm_config_from_storage + get_describe_backup_info + configsvr_name=$(echo "$describe_result" | jq -r '.replsets[] | select((.configsvr // false) == true or (.iscs // false) == true) | .name' | head -n 1) + shardsvr_names=$(echo "$describe_result" | jq -r '[.replsets[] | select(((.configsvr // false) != true) and ((.iscs // false) != true)) | .name] | join(",")') +fi -configsvr_name=$(echo "$describe_result" | jq -r '.replsets[] | select(.configsvr == true) | .name') echo "INFO: Config server replica set name: $configsvr_name" -shardsvr_names=$(echo "$describe_result" | jq -r '[.replsets[] | select(.configsvr != true) | .name] | join(",")') echo "INFO: Shard replica set names: $shardsvr_names" +if [ -z "$configsvr_name" ] || [ -z "$shardsvr_names" ]; then + echo "ERROR: Missing configsvr or shardsvr metadata for restore mapping." + exit 1 +fi + mappings="" IFS="," read -r -a shardsvr_array <<< "$shardsvr_names" shardsvr_count=${#shardsvr_array[@]} @@ -47,7 +57,6 @@ if [ $new_shardsvr_count -ne $shardsvr_count ]; then exit 1 fi for i in "${!shardsvr_array[@]}"; do - # Get the part before "@" in new_shardsvr_array if [ $shardsvr_count -gt 1 ]; then shard_name="$CLUSTER_NAME-${new_shardsvr_array[i]%%@*}" else @@ -60,17 +69,18 @@ for i in "${!shardsvr_array[@]}"; do mappings="$mappings,${shard_name}=${shardsvr_array[i]}" fi done -# If the config server name is not empty, add it to the mappings + echo "INFO: Mapping config server $configsvr_name to $CFG_SERVER_REPLICA_SET_NAME" mappings="$mappings,$CFG_SERVER_REPLICA_SET_NAME=$configsvr_name" echo "INFO: Shard mappings: $mappings" -process_restore_start_signal - -wait_for_other_operations - -restore_name=$(pbm restore $backup_name --mongodb-uri "$PBM_MONGODB_URI" --replset-remapping "$mappings" -o json | jq -r '.name') +echo "INFO: Starting syncer physical restore..." +if ! restore_result=$(syncerctl_cmd restore start --backup-name "$backup_name" --type physical --replset-remapping "$mappings" 2>&1); then + echo "ERROR: Syncer restore start failed: $restore_result" + exit 1 +fi +echo "INFO: Syncer restore start result: $restore_result" -wait_for_restoring +wait_for_syncer_restore_completion -process_restore_end_signal +echo "INFO: Restore completed." diff --git a/addons/mongodb/dataprotection/shard-pbm-pitr-backup.sh b/addons/mongodb/dataprotection/shard-pbm-pitr-backup.sh index 9ca07e1dce..9a727f8199 100644 --- a/addons/mongodb/dataprotection/shard-pbm-pitr-backup.sh +++ b/addons/mongodb/dataprotection/shard-pbm-pitr-backup.sh @@ -1,67 +1,79 @@ #!/bin/bash -# When doing point-in-time recovery for deployments with sharded collections, PBM only writes data to existing ones and doesn’t support creating new collections. +# When doing point-in-time recovery for deployments with sharded collections, PBM only writes data to existing ones and does not support creating new collections. # Therefore, whenever you create a new sharded collection, make a new backup for it to be included there. Ref: https://docs.percona.com/percona-backup-mongodb/usage/pitr-physical.html#post-restore-steps set -e set -o pipefail -export PATH="$PATH:$DP_DATASAFED_BIN_PATH" +export PATH="$PATH:$DP_DATASAFED_BIN_PATH:${MOUNT_DIR}/tmp/bin" export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" trap handle_exit EXIT function enable_pitr() { - local current_pitr_conf=$(pbm config --mongodb-uri "$PBM_MONGODB_URI" -o json | jq -r '.pitr') - local current_pitr_enabled=$(echo $current_pitr_conf | jq -r '.enabled') - local current_oplog_span_min=$(echo $current_pitr_conf | jq -r '.oplogSpanMin') - local current_pitr_compression=$(echo $current_pitr_conf | jq -r '.compression') + local current_pitr_conf + current_pitr_conf=$(syncerctl_cmd pitr status) + local current_pitr_enabled + local current_oplog_span_min + local current_pitr_compression + current_pitr_enabled=$(echo "$current_pitr_conf" | jq -r '.enabled // false') + current_oplog_span_min=$(echo "$current_pitr_conf" | jq -r '.oplog_span_min // empty') + current_pitr_compression=$(echo "$current_pitr_conf" | jq -r '.compression // empty') if [ "$current_pitr_enabled" != "true" ] || [ "$current_oplog_span_min" != "$PBM_OPLOG_SPAN_MIN_MINUTES" ] || [ "$current_pitr_compression" != "$PBM_COMPRESSION" ]; then - echo "INFO: Pitr config is not equal to the current config, updating..." + echo "INFO: PITR config is not equal to the desired config, updating through syncer..." wait_for_other_operations "backup" - - pbm config --mongodb-uri "$PBM_MONGODB_URI" --set pitr.enabled=true --set pitr.oplogSpanMin=$PBM_OPLOG_SPAN_MIN_MINUTES --set pitr.compression=$PBM_COMPRESSION - echo "INFO: Pitr config updated." + syncerctl_cmd pitr enable --oplog-span-min "$PBM_OPLOG_SPAN_MIN_MINUTES" --compression "$PBM_COMPRESSION" + echo "INFO: PITR config updated." fi } function disable_pitr() { - echo "INFO: Disabling Pitr..." - pbm config --set pitr.enabled=false --mongodb-uri "$PBM_MONGODB_URI" - echo "INFO: Pitr disabled." + echo "INFO: Disabling PITR through syncer..." + syncerctl_cmd pitr disable + echo "INFO: PITR disabled." } function upload_continuous_backup_info() { - local status_result=$(pbm status --mongodb-uri "$PBM_MONGODB_URI" -o json) + local status_result + status_result=$(syncerctl_cmd pitr chunks) echo "INFO: Uploading continuous backup info..." - local pitr_chunks_result=$(echo "$status_result" | jq -r '.backups.pitrChunks') echo "INFO: Continuous backup result:" - echo "$(echo $pitr_chunks_result | jq)" - local pitr_chunks_arr=$(echo "$pitr_chunks_result" | jq -r '.pitrChunks') + echo "$(echo "$status_result" | jq)" + local pitr_chunks_arr + pitr_chunks_arr=$(echo "$status_result" | jq -r '.pitrChunks') if [ -z "$pitr_chunks_arr" ] || [ "$pitr_chunks_arr" = "null" ] || [ "$pitr_chunks_arr" = "[]" ]; then echo "INFO: No oplog found." return fi - local filtered_sorted_chunks=$(echo "$pitr_chunks_arr" | jq 'map(select(.noBaseSnapshot != true)) | sort_by(.range.end)') + local filtered_sorted_chunks + filtered_sorted_chunks=$(echo "$pitr_chunks_arr" | jq 'map(select(.noBaseSnapshot != true)) | sort_by(.range.end)') if [ -z "$filtered_sorted_chunks" ] || [ "$filtered_sorted_chunks" = "null" ] || [ "$filtered_sorted_chunks" = "[]" ]; then echo "INFO: No oplog found." return fi - local last_chunk=$(echo "$filtered_sorted_chunks" | jq -r '.[-1].range') - local start_unix_time=$(echo "$last_chunk" | jq -r '.start') - local end_unix_time=$(echo "$last_chunk" | jq -r '.end') - local start_time=$(date -u -d "@${start_unix_time}" +"%Y-%m-%dT%H:%M:%SZ") - local end_time=$(date -u -d "@${end_unix_time}" +"%Y-%m-%dT%H:%M:%SZ") - local total_size=$(echo "$pitr_chunks_result" | jq -r '.size') + local last_chunk + last_chunk=$(echo "$filtered_sorted_chunks" | jq -r '.[-1].range') + local start_unix_time + local end_unix_time + start_unix_time=$(echo "$last_chunk" | jq -r '.start') + end_unix_time=$(echo "$last_chunk" | jq -r '.end') + local start_time + local end_time + start_time=$(date -u -d "@${start_unix_time}" +"%Y-%m-%dT%H:%M:%SZ") + end_time=$(date -u -d "@${end_unix_time}" +"%Y-%m-%dT%H:%M:%SZ") + local total_size + total_size=$(echo "$status_result" | jq -r '.size // 0') local backup_type="continuous" - local extras=$(buildJsonString "" "backup_type" "$backup_type") + local extras + extras=$(buildJsonString "" "backup_type" "$backup_type") - local pitr_nodes=$(echo "$status_result" | jq -r '.pitr.nodes?[]?') + local pitr_nodes + pitr_nodes=$(echo "$status_result" | jq -r '.nodes?[]?') local shardsvr="" local configsvr="" while IFS= read -r node; do if [[ -n "$node" ]]; then - # Extract the text before the first "/" local node_name=${node%%/*} if [[ "$node" == *"config"* ]]; then configsvr="$node_name" @@ -74,11 +86,16 @@ function upload_continuous_backup_info() { fi fi done <<< "$pitr_nodes" - if [ -z "$shardsvr" ]; then - shardsvr=$(echo "$status_result" | jq -r '[.cluster[].rs | select(contains("config-server") | not)] | join(",")') - fi - if [ -z "$configsvr" ]; then - configsvr=$(echo "$status_result" | jq -r '[.cluster[].rs | select(contains("config-server"))] | join(",")') + if [ -z "$shardsvr" ] || [ -z "$configsvr" ]; then + echo "INFO: PITR node list is incomplete, falling back to pbm status cluster metadata." + local pbm_status_result + pbm_status_result=$(pbm status --mongodb-uri "$PBM_MONGODB_URI" -o json) + if [ -z "$shardsvr" ]; then + shardsvr=$(echo "$pbm_status_result" | jq -r '[.cluster[]? | .rs | select(contains("config-server") | not)] | join(",")') + fi + if [ -z "$configsvr" ]; then + configsvr=$(echo "$pbm_status_result" | jq -r '[.cluster[]? | .rs | select(contains("config-server"))] | join(",")') + fi fi extras=$(buildJsonString $extras "shardsvr" "$shardsvr") extras=$(buildJsonString $extras "configsvr" "$configsvr") @@ -89,7 +106,6 @@ function upload_continuous_backup_info() { function purge_pitr_chunks() { current_time=$(date +%s) - # Initialize last_global_purge_time if not set if [ -z "$last_global_purge_time" ]; then last_global_purge_time=0 fi @@ -100,34 +116,32 @@ function purge_pitr_chunks() { return fi - # Update last purge time last_global_purge_time=$current_time - echo "INFO: Purging PBM chunks..." + echo "INFO: Purging PBM chunks through syncer..." wait_for_other_operations "backup" - pbm config --force-resync --mongodb-uri "$PBM_MONGODB_URI" - - # resync wait flag might don't work - wait_for_other_operations "backup" - - local pitr_chunks_result=$(pbm status --mongodb-uri "$PBM_MONGODB_URI" -o json | jq -r '.backups.pitrChunks') - local pitr_chunks_arr=$(echo "$pitr_chunks_result" | jq -r '.pitrChunks') + local status_result + local pitr_chunks_arr + status_result=$(syncerctl_cmd pitr chunks) + pitr_chunks_arr=$(echo "$status_result" | jq -r '.pitrChunks') if [ -z "$pitr_chunks_arr" ] || [ "$pitr_chunks_arr" = "null" ] || [ "$pitr_chunks_arr" = "[]" ]; then - echo "INFO: No no base snapshot chunks found." + echo "INFO: No no-base-snapshot chunks found." return fi - local filtered_sorted_chunks=$(echo "$pitr_chunks_arr" | jq -e 'map(select(.noBaseSnapshot == true)) | sort_by(.range.end)') + local filtered_sorted_chunks + filtered_sorted_chunks=$(echo "$pitr_chunks_arr" | jq -e 'map(select(.noBaseSnapshot == true)) | sort_by(.range.end)') if [ -z "$filtered_sorted_chunks" ] || [ "$filtered_sorted_chunks" = "null" ] || [ "$filtered_sorted_chunks" = "[]" ]; then - echo "INFO: No no base snapshot chunks." + echo "INFO: No no-base-snapshot chunks." return fi - echo "INFO: No base snapshot chunks:" - echo "$(echo $filtered_sorted_chunks | jq)" - local first_chunk_end=$(echo "$filtered_sorted_chunks" | jq -r '.[0].range.end') + echo "INFO: No-base-snapshot chunks:" + echo "$(echo "$filtered_sorted_chunks" | jq)" + local first_chunk_end + first_chunk_end=$(echo "$filtered_sorted_chunks" | jq -r '.[0].range.end') purge_time=$(date -u -d "@${first_chunk_end}" +"%Y-%m-%dT%H:%M:%S") - pbm cleanup --older-than $purge_time --mongodb-uri "$PBM_MONGODB_URI" --wait --yes - echo "INFO: No base snapshot chunks cleaned up." + syncerctl_cmd pitr cleanup --older-than "$purge_time" + echo "INFO: No-base-snapshot chunks cleanup requested." } export_pbm_env_vars @@ -138,15 +152,13 @@ export_logs_start_time_env trap handle_pitr_exit EXIT -sync_pbm_storage_config - -sync_pbm_config_from_storage +# Apply storage once. Re-applying the storage file in the loop clears PBM PITR settings +# and restarts slicing before chunks can mature. +configure_syncer_backup while true; do wait_for_other_operations "backup" - sync_pbm_storage_config - enable_pitr purge_pitr_chunks diff --git a/addons/mongodb/dataprotection/shard-pbm-pitr-restore.sh b/addons/mongodb/dataprotection/shard-pbm-pitr-restore.sh index 49ae89c743..49c683e870 100644 --- a/addons/mongodb/dataprotection/shard-pbm-pitr-restore.sh +++ b/addons/mongodb/dataprotection/shard-pbm-pitr-restore.sh @@ -14,15 +14,18 @@ trap handle_restore_exit EXIT wait_for_other_operations -sync_pbm_storage_config - -sync_pbm_config_from_storage +ensure_restore_coord_storage_config extras=$(cat /dp_downward/status_extras) -configsvr_name=$(echo "$extras" | jq -r '.[0].configsvr') +configsvr_name=$(echo "$extras" | jq -r '.[0].configsvr // empty') echo "INFO: Config server replica set name: $configsvr_name" -shardsvr_names=$(echo "$extras" | jq -r '.[0].shardsvr') +shardsvr_names=$(echo "$extras" | jq -r '.[0].shardsvr // empty') echo "INFO: Shard replica set names: $shardsvr_names" +if [ -z "$configsvr_name" ] || [ -z "$shardsvr_names" ]; then + echo "ERROR: Missing configsvr or shardsvr metadata for PITR restore mapping." + exit 1 +fi + mappings="" IFS="," read -r -a shardsvr_array <<< "$shardsvr_names" shardsvr_count=${#shardsvr_array[@]} @@ -37,7 +40,6 @@ if [ $new_shardsvr_count -ne $shardsvr_count ]; then exit 1 fi for i in "${!shardsvr_array[@]}"; do - # Get the part before "@" in new_shardsvr_array if [ $shardsvr_count -gt 1 ]; then shard_name="$CLUSTER_NAME-${new_shardsvr_array[i]%%@*}" else @@ -50,25 +52,21 @@ for i in "${!shardsvr_array[@]}"; do mappings="$mappings,${shard_name}=${shardsvr_array[i]}" fi done -# If the config server name is not empty, add it to the mappings + echo "INFO: Mapping config server $configsvr_name to $CFG_SERVER_REPLICA_SET_NAME" mappings="$mappings,$CFG_SERVER_REPLICA_SET_NAME=$configsvr_name" echo "INFO: Shard mappings: $mappings" -process_restore_start_signal - recovery_target_time=$(date -d "@${DP_RESTORE_TIMESTAMP}" +"%Y-%m-%dT%H:%M:%S") echo "INFO: Recovery target time: $recovery_target_time" +echo "INFO: Starting syncer PITR restore..." +if ! restore_result=$(syncerctl_cmd restore start --pitr-target "$recovery_target_time" --type physical --replset-remapping "$mappings" 2>&1); then + echo "ERROR: Syncer restore start failed: $restore_result" + exit 1 +fi +echo "INFO: Syncer restore start result: $restore_result" -echo "INFO: Starting restore..." - -wait_for_other_operations - -restore_name=$(pbm restore --time="$recovery_target_time" --mongodb-uri "$PBM_MONGODB_URI" --replset-remapping "$mappings" -o json | jq -r '.name') - -wait_for_restoring - -process_restore_end_signal +wait_for_syncer_restore_completion echo "INFO: Restore completed." diff --git a/addons/mongodb/dataprotection/shard-pbm-restore-post-ready.sh b/addons/mongodb/dataprotection/shard-pbm-restore-post-ready.sh new file mode 100644 index 0000000000..330bcb5711 --- /dev/null +++ b/addons/mongodb/dataprotection/shard-pbm-restore-post-ready.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e +set -o pipefail + +wait_for_mongos_router_ready + +echo "INFO: Post-restore mongos router is ready." diff --git a/addons/mongodb/scripts/mongodb-member-leave.sh b/addons/mongodb/scripts/mongodb-member-leave.sh index b742d7b784..be355b0889 100644 --- a/addons/mongodb/scripts/mongodb-member-leave.sh +++ b/addons/mongodb/scripts/mongodb-member-leave.sh @@ -1,8 +1,41 @@ #!/bin/bash +set -euo pipefail -if [[ "$(timeout 5s /tools/syncerctl getrole)" == "primary" ]]; then - echo "current member role is primary." - exit 1 +member_name="${KB_LEAVE_MEMBER_POD_NAME:-}" +if [ -z "$member_name" ]; then + echo "KB_LEAVE_MEMBER_POD_NAME is required" >&2 + exit 1 fi -/tools/syncerctl leave --instance "$KB_LEAVE_MEMBER_POD_NAME" \ No newline at end of file +syncerctl_bin="${SYNCERCTL_BIN:-/tools/syncerctl}" +main_port="${SYNCER_MAIN_SERVICE_PORT:-3601}" +pbm_agent_port="${PBM_AGENT_SYNCER_SERVICE_PORT:-3361}" +call_timeout="${SYNCER_MEMBER_LEAVE_CALL_TIMEOUT:-60s}" + +call_leave() { + local port="$1" + local target="$2" + local output + local exit_code + echo "calling ${target} memberLeave for ${member_name} on 127.0.0.1:${port}" + set +e + output=$(timeout "$call_timeout" "$syncerctl_bin" --host 127.0.0.1 --port "$port" leave --instance "$member_name" 2>&1) + exit_code=$? + set -e + echo "$output" + if [ "$exit_code" -ne 0 ]; then + echo "${target} memberLeave command failed with exit code ${exit_code}" >&2 + return "$exit_code" + fi + if echo "$output" | grep -q "leave member failed"; then + echo "${target} memberLeave failed" >&2 + return 1 + fi + if ! echo "$output" | grep -q "leave member success"; then + echo "${target} memberLeave returned unexpected output" >&2 + return 1 + fi +} + +call_leave "$main_port" "mongodb" +call_leave "$pbm_agent_port" "pbm-agent" diff --git a/addons/mongodb/scripts/mongodb-mongos-setup.sh b/addons/mongodb/scripts/mongodb-mongos-setup.sh index 068dd078b2..c5444df39c 100644 --- a/addons/mongodb/scripts/mongodb-mongos-setup.sh +++ b/addons/mongodb/scripts/mongodb-mongos-setup.sh @@ -10,11 +10,4 @@ export PATH=$MONGODB_ROOT/tmp/bin:$PATH cfg_server_endpoints="$(generate_endpoints "$CFG_SERVER_POD_FQDN_LIST" "$CFG_SERVER_INTERNAL_PORT")" process="mongos --bind_ip_all --port $MONGOS_PORT --configdb $CFG_SERVER_REPLICA_SET_NAME/$cfg_server_endpoints --config /etc/mongodb/mongos.conf" - -boot_or_enter_restore "$process" - -$process & - -process_restore_signal "$process" "start" - -process_restore_signal "$process" "end" \ No newline at end of file +exec $process diff --git a/addons/mongodb/scripts/mongodb-shard-setup.sh b/addons/mongodb/scripts/mongodb-shard-setup.sh index cf6dcfe627..11765a77c0 100644 --- a/addons/mongodb/scripts/mongodb-shard-setup.sh +++ b/addons/mongodb/scripts/mongodb-shard-setup.sh @@ -11,14 +11,4 @@ export PATH=$MONGODB_ROOT/tmp/bin:$PATH . "/scripts/mongodb-common.sh" process="mongod --bind_ip_all --port $PORT --replSet $CLUSTER_COMPONENT_NAME --config /etc/mongodb/mongodb.conf" -boot_or_enter_restore "$process" - -echo "INFO: Startup backup agent for restore." -pbm-agent-entrypoint >> $MONGODB_ROOT/tmp/pbm_agent_restore.log 2>&1 & - -echo "INFO: Start mongodb for restore." -$process & - -process_restore_signal "$process" "start" - -process_restore_signal "$process" "end" +exec $process diff --git a/addons/mongodb/templates/actionset-rs-pbm-physical.yaml b/addons/mongodb/templates/actionset-rs-pbm-physical.yaml index 9059fa9ec3..417d32204f 100644 --- a/addons/mongodb/templates/actionset-rs-pbm-physical.yaml +++ b/addons/mongodb/templates/actionset-rs-pbm-physical.yaml @@ -47,13 +47,6 @@ spec: - | {{- .Files.Get "dataprotection/pbm-backup-delete.sh" | nindent 10 }} restore: - prepareData: - image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:$(PBM_IMAGE_TAG) - command: - - bash - - -c - - | - {{- .Files.Get "dataprotection/rs-pbm-restore-prepare.sh" | nindent 8 }} postReady: - job: image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:$(PBM_IMAGE_TAG) diff --git a/addons/mongodb/templates/actionset-rs-pbm-pitr.yaml b/addons/mongodb/templates/actionset-rs-pbm-pitr.yaml index dd24322616..670d1ce76a 100644 --- a/addons/mongodb/templates/actionset-rs-pbm-pitr.yaml +++ b/addons/mongodb/templates/actionset-rs-pbm-pitr.yaml @@ -59,13 +59,6 @@ spec: {{- .Files.Get "dataprotection/pbm-backup-delete.sh" | nindent 10 }} restore: - prepareData: - image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:$(PBM_IMAGE_TAG) - command: - - bash - - -c - - | - {{- .Files.Get "dataprotection/rs-pbm-restore-prepare.sh" | nindent 8 }} postReady: - job: image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:$(PBM_IMAGE_TAG) diff --git a/addons/mongodb/templates/actionset-shard-pbm-physical.yaml b/addons/mongodb/templates/actionset-shard-pbm-physical.yaml index bf86232f54..ddc2175ce9 100644 --- a/addons/mongodb/templates/actionset-shard-pbm-physical.yaml +++ b/addons/mongodb/templates/actionset-shard-pbm-physical.yaml @@ -72,4 +72,13 @@ spec: - -c - | {{- .Files.Get "dataprotection/common-scripts.sh" | nindent 12 }} - {{- .Files.Get "dataprotection/shard-pbm-full-restore.sh" | nindent 12 }} \ No newline at end of file + {{- .Files.Get "dataprotection/shard-pbm-full-restore.sh" | nindent 12 }} + - job: + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:$(PSM_IMAGE_TAG) + runOnTargetPodNode: false + command: + - bash + - -c + - | + {{- .Files.Get "dataprotection/common-scripts.sh" | nindent 12 }} + {{- .Files.Get "dataprotection/shard-pbm-restore-post-ready.sh" | nindent 12 }} \ No newline at end of file diff --git a/addons/mongodb/templates/actionset-shard-pbm-pitr.yaml b/addons/mongodb/templates/actionset-shard-pbm-pitr.yaml index d3e65ddf3b..6c52fb18fc 100644 --- a/addons/mongodb/templates/actionset-shard-pbm-pitr.yaml +++ b/addons/mongodb/templates/actionset-shard-pbm-pitr.yaml @@ -84,4 +84,13 @@ spec: - -c - | {{- .Files.Get "dataprotection/common-scripts.sh" | nindent 12 }} - {{- .Files.Get "dataprotection/shard-pbm-pitr-restore.sh" | nindent 12 }} \ No newline at end of file + {{- .Files.Get "dataprotection/shard-pbm-pitr-restore.sh" | nindent 12 }} + - job: + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:$(PSM_IMAGE_TAG) + runOnTargetPodNode: false + command: + - bash + - -c + - | + {{- .Files.Get "dataprotection/common-scripts.sh" | nindent 12 }} + {{- .Files.Get "dataprotection/shard-pbm-restore-post-ready.sh" | nindent 12 }} \ No newline at end of file diff --git a/addons/mongodb/templates/cmpd-config-server.yaml b/addons/mongodb/templates/cmpd-config-server.yaml index 3e5f667733..cfe0403cf3 100644 --- a/addons/mongodb/templates/cmpd-config-server.yaml +++ b/addons/mongodb/templates/cmpd-config-server.yaml @@ -65,6 +65,7 @@ spec: - /bin/sh - -c - /scripts/mongodb-member-leave.sh > /tmp/member-leave.log 2>&1 + timeoutSeconds: -1 switchover: exec: container: mongodb @@ -78,17 +79,20 @@ spec: runAsUser: 0 initContainers: - command: - - cp - - -r - - /bin/syncer - - /bin/syncerctl - - /tools/ + - sh + - -c + - | + mkdir -p {{ .Values.dataMountPath }}/tmp/bin + cp -r /bin/syncer /bin/syncerctl /tools/ + cp -r /bin/syncerctl {{ .Values.dataMountPath }}/tmp/bin image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} name: init-syncer volumeMounts: - mountPath: /tools name: tools + - mountPath: {{ .Values.dataMountPath }} + name: data - command: - sh - -c @@ -154,6 +158,16 @@ spec: value: /tools/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - name: KB_SERVICE_CHARACTER_TYPE value: mongodb + - name: SYNCER_BACKUP_ENABLED + value: "true" + - name: PBM_BIN + value: /tools/pbm + - name: PBM_AGENT_BIN + value: /tools/pbm-agent + - name: PBM_RESTORE_AGENT_LOG_PATH + value: {{ .Values.dataMountPath }}/tmp/pbm-agent-restore.log + - name: PBM_DATA_MOUNT_POINT + value: {{ .Values.dataMountPath }} - name: SERVICE_PORT value: $(KB_SERVICE_PORT) - name: MONGODB_ROOT_USER @@ -195,13 +209,47 @@ spec: - name: mongodb-backup-agent image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:{{ .Values.image.percona.backup.tag }} imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} + ports: + - name: pbm-agent-ha + protocol: TCP + containerPort: 3361 command: - - /bin/bash - - -c + - /tools/syncer + - -- - /scripts/backup-agent-setup.sh env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP - name: PATH value: /tools/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: KB_SERVICE_CHARACTER_TYPE + value: mongodb + - name: KB_WORKLOAD_TYPE + value: pbm-agent + - name: SYNCER_IGNORE_HA + value: "true" + - name: SYNCER_BACKUP_ENABLED + value: "true" + - name: SYNCER_SERVICE_PORT + value: "3361" - name: PBM_MONGODB_REPLICA_SET value: $(CLUSTER_COMPONENT_NAME) - name: PBM_AGENT_SIDECAR @@ -330,22 +378,11 @@ spec: port: name: mongodb - name: MONGOS_INTERNAL_SVC_NAME - valueFrom: - serviceVarRef: - name: internal - compDef: {{ include "mongos.compDefName" . }} - optional: false - host: Required + value: "$(CLUSTER_NAME)-mongos-internal" - name: MONGOS_INTERNAL_PORT - valueFrom: - serviceVarRef: - name: internal - compDef: {{ include "mongos.compDefName" . }} - optional: false - port: - name: mongos + value: "27018" - name: MONGOS_INTERNAL_HOST - value: "$(MONGOS_INTERNAL_SVC_NAME).$(CLUSTER_NAMESPACE).svc.{{ .Values.clusterDomain }}" + value: "$(CLUSTER_NAME)-mongos-internal.$(CLUSTER_NAMESPACE).svc.{{ .Values.clusterDomain }}" - name: MONGODB_SHARD_REPLICA_SET_NAME_LIST valueFrom: componentVarRef: diff --git a/addons/mongodb/templates/cmpd-mongodb-shard.yaml b/addons/mongodb/templates/cmpd-mongodb-shard.yaml index eaa32f8f07..f694f018e3 100644 --- a/addons/mongodb/templates/cmpd-mongodb-shard.yaml +++ b/addons/mongodb/templates/cmpd-mongodb-shard.yaml @@ -69,6 +69,13 @@ spec: verbs: - get - list + - apiGroups: + - "dataprotection.kubeblocks.io" + resources: + - backups + verbs: + - get + - list exporter: containerName: exporter scrapePath: /metrics @@ -89,6 +96,7 @@ spec: - /bin/sh - -c - /scripts/mongodb-member-leave.sh > /tmp/member-leave.log 2>&1 + timeoutSeconds: -1 switchover: exec: container: mongodb @@ -106,6 +114,7 @@ spec: targetPodSelector: Role matchingKey: primary preCondition: RuntimeReady + timeoutSeconds: -1 retryPolicy: maxRetries: 10 runtime: @@ -114,17 +123,20 @@ spec: runAsUser: 0 initContainers: - command: - - cp - - -r - - /bin/syncer - - /bin/syncerctl - - /tools/ + - sh + - -c + - | + mkdir -p {{ .Values.dataMountPath }}/tmp/bin + cp -r /bin/syncer /bin/syncerctl /tools/ + cp -r /bin/syncerctl {{ .Values.dataMountPath }}/tmp/bin image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} name: init-syncer volumeMounts: - mountPath: /tools name: tools + - mountPath: {{ .Values.dataMountPath }} + name: data - command: - sh - -c @@ -190,6 +202,16 @@ spec: value: /tools/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - name: KB_SERVICE_CHARACTER_TYPE value: mongodb + - name: SYNCER_BACKUP_ENABLED + value: "true" + - name: PBM_BIN + value: /tools/pbm + - name: PBM_AGENT_BIN + value: /tools/pbm-agent + - name: PBM_RESTORE_AGENT_LOG_PATH + value: {{ .Values.dataMountPath }}/tmp/pbm-agent-restore.log + - name: PBM_DATA_MOUNT_POINT + value: {{ .Values.dataMountPath }} - name: SERVICE_PORT value: $(KB_SERVICE_PORT) - name: MONGODB_ROOT_USER @@ -231,13 +253,47 @@ spec: - name: mongodb-backup-agent image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:{{ .Values.image.percona.backup.tag }} imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} + ports: + - name: pbm-agent-ha + protocol: TCP + containerPort: 3361 command: - - /bin/bash - - -c + - /tools/syncer + - -- - /scripts/backup-agent-setup.sh env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP - name: PATH value: /tools/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: KB_SERVICE_CHARACTER_TYPE + value: mongodb + - name: KB_WORKLOAD_TYPE + value: pbm-agent + - name: SYNCER_IGNORE_HA + value: "true" + - name: SYNCER_BACKUP_ENABLED + value: "true" + - name: SYNCER_SERVICE_PORT + value: "3361" - name: PBM_AGENT_MONGODB_USERNAME value: $(MONGODB_ADMIN_USER) - name: PBM_AGENT_MONGODB_PASSWORD @@ -347,22 +403,11 @@ spec: optional: false password: Required - name: MONGOS_INTERNAL_SVC_NAME - valueFrom: - serviceVarRef: - name: internal - compDef: {{ include "mongos.compDefName" . }} - optional: false - host: Required + value: "$(CLUSTER_NAME)-mongos-internal" - name: MONGOS_INTERNAL_PORT - valueFrom: - serviceVarRef: - name: internal - compDef: {{ include "mongos.compDefName" . }} - optional: false - port: - name: mongos + value: "27018" - name: MONGOS_INTERNAL_HOST - value: "$(MONGOS_INTERNAL_SVC_NAME).$(CLUSTER_NAMESPACE).svc.{{ .Values.clusterDomain }}" + value: "$(CLUSTER_NAME)-mongos-internal.$(CLUSTER_NAMESPACE).svc.{{ .Values.clusterDomain }}" - name: MONGODB_POD_FQDN_LIST valueFrom: componentVarRef: diff --git a/addons/mongodb/templates/cmpd-mongos.yaml b/addons/mongodb/templates/cmpd-mongos.yaml index 52e431a9b0..5519b20c66 100644 --- a/addons/mongodb/templates/cmpd-mongos.yaml +++ b/addons/mongodb/templates/cmpd-mongos.yaml @@ -54,6 +54,17 @@ spec: # percona-server-mongodb needs run as root to have privilege to access to keyfile runAsUser: 0 initContainers: + - command: + - sh + - -c + - | + cp -r /bin/syncer /bin/syncerctl /kubeblocks/ + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.syncer.repository }}:{{ .Values.image.syncer.tag }} + imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} + name: init-syncer + volumeMounts: + - mountPath: /kubeblocks + name: kubeblocks - command: - sh - -c @@ -74,9 +85,12 @@ spec: - name: mongos protocol: TCP containerPort: 27017 + - name: ha + protocol: TCP + containerPort: 3601 command: - - bash - - -c + - /kubeblocks/syncer + - -- - /scripts/mongodb-mongos-setup.sh env: - name: POD_NAME @@ -101,6 +115,14 @@ spec: fieldPath: status.podIP - name: PATH value: /kubeblocks/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: KB_SERVICE_CHARACTER_TYPE + value: mongodb + - name: KB_WORKLOAD_TYPE + value: mongos + - name: SYNCER_IGNORE_HA + value: "true" + - name: SYNCER_BACKUP_ENABLED + value: "true" - name: SERVICE_PORT value: $(KB_SERVICE_PORT) - name: MONGODB_ROOT_USER @@ -130,7 +152,7 @@ spec: - -c - | client=$(mongosh --version 1>/dev/null&&echo mongosh||echo mongo) - $client --port $SERVICE_PORT -u $MONGOS_USER -p $MONGOS_PASSWORD --authenticationDatabase admin --eval "db.adminCommand('ping')" + exec timeout -k 1s 3s "$client" --quiet --port "$SERVICE_PORT" -u "$MONGOS_USER" -p "$MONGOS_PASSWORD" --authenticationDatabase admin --eval "db.adminCommand('ping')" - name: exporter imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} command: @@ -192,6 +214,15 @@ spec: componentName: Required - name: KB_SERVICE_PORT value: "27017" + - name: SYNCER_SERVICE_PORT + valueFrom: + hostNetworkVarRef: + optional: true + container: + name: mongos + port: + name: ha + option: Optional - name: CFG_SERVER_REPLICA_SET_NAME valueFrom: componentVarRef: @@ -233,4 +264,5 @@ spec: containerPorts: - container: mongos ports: - - mongos \ No newline at end of file + - mongos + - ha \ No newline at end of file diff --git a/addons/mongodb/templates/cmpd.yaml b/addons/mongodb/templates/cmpd.yaml index 4a9ea506ee..ef3edda655 100644 --- a/addons/mongodb/templates/cmpd.yaml +++ b/addons/mongodb/templates/cmpd.yaml @@ -88,6 +88,7 @@ spec: - /bin/sh - -c - /scripts/mongodb-member-leave.sh > /tmp/member-leave.log 2>&1 + timeoutSeconds: -1 switchover: exec: container: mongodb @@ -101,16 +102,19 @@ spec: runAsUser: 0 initContainers: - command: - - cp - - -r - - /bin/syncer - - /bin/syncerctl - - /tools/ + - sh + - -c + - | + mkdir -p {{ .Values.dataMountPath }}/tmp/bin + cp -r /bin/syncer /bin/syncerctl /tools/ + cp -r /bin/syncerctl {{ .Values.dataMountPath }}/tmp/bin imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} name: init-syncer volumeMounts: - mountPath: /tools name: tools + - mountPath: {{ .Values.dataMountPath }} + name: data - command: - sh - -c @@ -175,6 +179,16 @@ spec: value: /tools/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - name: KB_SERVICE_CHARACTER_TYPE value: mongodb + - name: SYNCER_BACKUP_ENABLED + value: "true" + - name: PBM_BIN + value: /tools/pbm + - name: PBM_AGENT_BIN + value: /tools/pbm-agent + - name: PBM_RESTORE_AGENT_LOG_PATH + value: {{ .Values.dataMountPath }}/tmp/pbm-agent-restore.log + - name: PBM_DATA_MOUNT_POINT + value: {{ .Values.dataMountPath }} - name: SERVICE_PORT value: $(KB_SERVICE_PORT) - name: MONGODB_ROOT_USER @@ -214,18 +228,47 @@ spec: - name: mongodb-backup-agent image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.percona.backup.repository }}:{{ .Values.image.percona.backup.tag }} imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} + ports: + - name: pbm-agent-ha + protocol: TCP + containerPort: 3361 command: - - /bin/bash - - -c - - | - while [ -f $PBM_DATA_MOUNT_POINT/tmp/mongodb_pbm.backup ]; do - echo "Waiting for restore..."; - sleep 5; - done - exec pbm-agent-entrypoint + - /tools/syncer + - -- + - /scripts/backup-agent-setup.sh env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP - name: PATH value: /tools/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: KB_SERVICE_CHARACTER_TYPE + value: mongodb + - name: KB_WORKLOAD_TYPE + value: pbm-agent + - name: SYNCER_IGNORE_HA + value: "true" + - name: SYNCER_BACKUP_ENABLED + value: "true" + - name: SYNCER_SERVICE_PORT + value: "3361" - name: PBM_AGENT_MONGODB_USERNAME value: $(MONGODB_USER) - name: PBM_AGENT_MONGODB_PASSWORD @@ -245,6 +288,8 @@ spec: name: data - name: scripts mountPath: /scripts + - mountPath: /tools + name: tools - name: exporter imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} command: diff --git a/addons/mongodb/templates/cmpv-mongos.yaml b/addons/mongodb/templates/cmpv-mongos.yaml index d1a9253940..f00cef1407 100644 --- a/addons/mongodb/templates/cmpv-mongos.yaml +++ b/addons/mongodb/templates/cmpv-mongos.yaml @@ -27,6 +27,7 @@ spec: serviceVersion: {{ index . 1 }} images: mongos: {{ $imageRegistry }}/{{ $.Values.image.repository }}:{{ index . 2 }} + init-syncer: {{ $imageRegistry }}/{{ $.Values.image.syncer.repository }}:{{ $.Values.image.syncer.tag }} init-kubectl: {{ $imageRegistry }}/{{ $.Values.image.kubectl.repository }}:{{ $.Values.image.kubectl.tag }} exporter: {{ $imageRegistry }}/{{ $.Values.image.exporter.repository }}:{{ $.Values.image.exporter.tag }} {{- end }} diff --git a/addons/mongodb/templates/script-template.yaml b/addons/mongodb/templates/script-template.yaml index 3d654e820a..463f2772c6 100644 --- a/addons/mongodb/templates/script-template.yaml +++ b/addons/mongodb/templates/script-template.yaml @@ -17,9 +17,11 @@ data: DATA_VOLUME={{ .Values.dataMountPath }} {{- .Files.Get "scripts/replicaset-setup.tpl" | nindent 4 }} - mongodb-common.sh: |- - {{- .Files.Get "scripts/mongodb-common.sh" | nindent 4 }} + backup-agent-setup.sh: |- + {{- .Files.Get "scripts/backup-agent-setup.sh" | nindent 4 }} mongodb-member-leave.sh: |- {{- .Files.Get "scripts/mongodb-member-leave.sh" | nindent 4 }} + mongodb-common.sh: |- + {{- .Files.Get "scripts/mongodb-common.sh" | nindent 4 }} mongodb-switchover.sh: |- {{- .Files.Get "scripts/mongodb-switchover.sh" | nindent 4 }} \ No newline at end of file