diff --git a/addons/mysql/scripts-ut-spec/orc_role_probe_spec.sh b/addons/mysql/scripts-ut-spec/orc_role_probe_spec.sh new file mode 100644 index 0000000000..fcbf43ca9d --- /dev/null +++ b/addons/mysql/scripts-ut-spec/orc_role_probe_spec.sh @@ -0,0 +1,107 @@ +# shellcheck shell=bash +# shellcheck disable=SC2034,SC2329 + +Describe "ORC role probe script tests" + Include ../scripts/orc-role-probe.sh + + setup_role_probe() { + export KB_AGENT_POD_NAME="mysql-0" + } + + cleanup_role_probe() { + unset KB_AGENT_POD_NAME + } + + Before 'setup_role_probe' + After 'cleanup_role_probe' + + It "publishes primary when Orchestrator identifies the local pod as master" + run_orc_role_probe() { + printf 'mysql-0:3306\n' + } + + When call probe_orchestrator_role + The status should be success + The output should equal "primary" + End + + It "does not parse successful client stderr noise as the master name" + run_orc_role_probe() { + printf 'transient client warning\n' >&2 + printf 'mysql-0:3306\n' + } + + When call probe_orchestrator_role + The status should be success + The output should equal "primary" + The error should include "transient client warning" + End + + It "publishes secondary when the local pod is in the replica list" + run_orc_role_probe() { + if [ "$2" = "which-cluster-master" ]; then + printf 'mysql-1:3306\n' + else + printf 'mysql-0:3306\nmysql-2:3306\n' + fi + } + + When call probe_orchestrator_role + The status should be success + The output should equal "secondary" + End + + It "fails instead of publishing an empty role when the master query fails" + run_orc_role_probe() { + printf 'orchestrator unavailable\n' + return 1 + } + + When call probe_orchestrator_role + The status should be failure + The error should include "cannot determine master" + The output should equal "" + End + + It "fails instead of publishing an empty role when the master output is empty" + run_orc_role_probe() { + return 0 + } + + When call probe_orchestrator_role + The status should be failure + The error should include "master query returned empty output" + The output should equal "" + End + + It "fails instead of publishing an empty role when the replica query fails" + run_orc_role_probe() { + if [ "$2" = "which-cluster-master" ]; then + printf 'mysql-1:3306\n' + return 0 + fi + printf 'replica query failed\n' + return 1 + } + + When call probe_orchestrator_role + The status should be failure + The error should include "cannot list replicas" + The output should equal "" + End + + It "fails instead of publishing an empty role when the pod is absent from topology" + run_orc_role_probe() { + if [ "$2" = "which-cluster-master" ]; then + printf 'mysql-1:3306\n' + else + printf 'mysql-2:3306\n' + fi + } + + When call probe_orchestrator_role + The status should be failure + The error should include "is absent from Orchestrator topology" + The output should equal "" + End +End diff --git a/addons/mysql/scripts-ut-spec/orc_switchover_spec.sh b/addons/mysql/scripts-ut-spec/orc_switchover_spec.sh new file mode 100644 index 0000000000..82379fd506 --- /dev/null +++ b/addons/mysql/scripts-ut-spec/orc_switchover_spec.sh @@ -0,0 +1,197 @@ +# shellcheck shell=bash +# shellcheck disable=SC2034,SC2329 + +Describe "ORC switchover script tests" + Include ../scripts/orc-switchover.sh + + Describe "MySQL read flag parsing" + It "recognizes writable flags after mysql client stderr noise" + output=$(printf '%s\n%s\n' \ + 'mysql: [Warning] Using a password on the command line interface can be insecure.' \ + '0 0') + + When call is_writable_mysql "$output" + The status should be success + End + + It "recognizes read-only flags after mysql client stderr noise" + output=$(printf '%s\n%s\n' \ + 'mysql: [Warning] Using a password on the command line interface can be insecure.' \ + '1 1') + + When call is_readonly_mysql "$output" + The status should be success + End + + It "rejects output without a read_only/super_read_only row" + output='mysql: [Warning] Using a password on the command line interface can be insecure.' + + When call is_writable_mysql "$output" + The status should be failure + End + End + + Describe "Switchover closure verification" + setup_switchover_verify() { + export KB_SWITCHOVER_CURRENT_NAME="mysql-0" + export KB_SWITCHOVER_CANDIDATE_NAME="mysql-1" + export MYSQL_ORC_SWITCHOVER_VERIFY_ATTEMPTS=20 + export MYSQL_ORC_SWITCHOVER_VERIFY_INTERVAL_SECONDS=0 + export MYSQL_ORC_SWITCHOVER_PRECHECK_TIMEOUT_SECONDS=3 + export MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS=40 + export MYSQL_ORC_SWITCHOVER_MYSQL_TIMEOUT_SECONDS=1 + export MYSQL_ORC_SWITCHOVER_MYSQL_CONNECT_TIMEOUT_SECONDS=1 + VERIFY_COUNTER_FILE=$(mktemp) + export VERIFY_COUNTER_FILE + printf '0\n' > "$VERIFY_COUNTER_FILE" + } + + cleanup_switchover_verify() { + rm -f "${VERIFY_COUNTER_FILE:-}" + unset KB_SWITCHOVER_CURRENT_NAME + unset KB_SWITCHOVER_CANDIDATE_NAME + unset MYSQL_ORC_SWITCHOVER_VERIFY_ATTEMPTS + unset MYSQL_ORC_SWITCHOVER_VERIFY_INTERVAL_SECONDS + unset MYSQL_ORC_SWITCHOVER_PRECHECK_TIMEOUT_SECONDS + unset MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS + unset MYSQL_ORC_SWITCHOVER_MYSQL_TIMEOUT_SECONDS + unset MYSQL_ORC_SWITCHOVER_MYSQL_CONNECT_TIMEOUT_SECONDS + unset VERIFY_COUNTER_FILE + unset ORC_SWITCHOVER_CLIENT_PID + unset ORC_SWITCHOVER_CLIENT_OUTPUT_FILE + unset ORC_SWITCHOVER_CLIENT_RC_FILE + unset ORC_SWITCHOVER_CLIENT_TEMP_DIR + unset ORC_SWITCHOVER_CLIENT_RC + unset ORC_SWITCHOVER_CLIENT_OUTPUT + } + + Before 'setup_switchover_verify' + After 'cleanup_switchover_verify' + + It "succeeds when readback converges inside the bounded verify window" + mysql_read_flags() { + local host="$1" + local count + if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then + count=$(cat "$VERIFY_COUNTER_FILE") + count=$((count + 1)) + printf '%s\n' "$count" > "$VERIFY_COUNTER_FILE" + if [ "$count" -lt 3 ]; then + printf '1 1\n' + return 0 + fi + printf '0 0\n' + return 0 + fi + printf '1 1\n' + } + + When call verify_switchover_closed_or_defer + The status should be success + The output should include "Switchover verified" + End + + It "uses raw parallel readback output for closure checks" + mysql_read_flags() { + local host="$1" + printf '%s\n' 'mysql: [Warning] Using a password on the command line interface can be insecure.' + if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then + printf '0 0\n' + return 0 + fi + printf '1 1\n' + } + + When call verify_switchover_closed_once + The status should be success + End + + It "classifies an unclosed readback window as retry-safe" + mysql_read_flags() { + local host="$1" + if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then + printf '1 1\n' + return 0 + fi + printf '1 1\n' + } + + When call verify_switchover_closed_or_defer + The status should be failure + The error should include "phase: post-switchover-not-converged" + The error should include "next-retry-safe: yes" + The error should include "verify-history:" + End + + It "accepts a non-zero orchestrator client result when same invocation verifies closure" + run_orchestrator_client_with_budget() { + printf 'client timed out\n' + return 124 + } + + mysql_read_flags() { + local host="$1" + if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then + printf '0 0\n' + return 0 + fi + printf '1 1\n' + } + + When call run_switchover_client_and_verify 40 -c graceful-master-takeover-auto -i "$KB_SWITCHOVER_CURRENT_NAME" -d "$KB_SWITCHOVER_CANDIDATE_NAME" + The status should be success + The output should include "Switchover command returned non-zero (124) but post-check observed the target topology." + The output should include "client timed out" + End + + It "keeps unclosed readback retry-safe with orchestrator client diagnostics" + run_orchestrator_client_with_budget() { + printf 'client timed out\n' + return 124 + } + + mysql_read_flags() { + local host="$1" + if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then + printf '1 1\n' + return 0 + fi + printf '1 1\n' + } + + When call run_switchover_client_and_verify 40 -c graceful-master-takeover-auto -i "$KB_SWITCHOVER_CURRENT_NAME" -d "$KB_SWITCHOVER_CANDIDATE_NAME" + The status should be failure + The error should include "phase: post-switchover-not-converged" + The error should include "orchestrator-client-rc: 124" + The error should include "client timed out" + The error should include "phase: orchestrator-command-failed" + End + + It "fails explicitly when the orchestrator client background wrapper cannot start" + start_orchestrator_client_background() { + ORC_SWITCHOVER_CLIENT_RC=1 + ORC_SWITCHOVER_CLIENT_OUTPUT="failed to create orchestrator client temp directory" + return 1 + } + + When call run_switchover_client_and_verify 40 -c graceful-master-takeover-auto -i "$KB_SWITCHOVER_CURRENT_NAME" -d "$KB_SWITCHOVER_CANDIDATE_NAME" + The status should be failure + The error should include "phase: orchestrator-client-start-failed" + The error should include "failed to create orchestrator client temp directory" + End + + It "does not reuse stale readback flags when its temp directory cannot be created" + SWITCHOVER_VERIFY_CANDIDATE_RAW="0 0" + SWITCHOVER_VERIFY_CURRENT_RAW="1 1" + mktemp() { + return 1 + } + + When call read_mysql_flags_pair "$KB_SWITCHOVER_CANDIDATE_NAME" "$KB_SWITCHOVER_CURRENT_NAME" + The status should be failure + The variable SWITCHOVER_VERIFY_CANDIDATE_RAW should equal "" + The variable SWITCHOVER_VERIFY_CURRENT_RAW should equal "" + The variable SWITCHOVER_VERIFY_CANDIDATE_FLAGS should include "failed to create readback temp directory" + End + End +End diff --git a/addons/mysql/scripts/orc-role-probe.sh b/addons/mysql/scripts/orc-role-probe.sh new file mode 100644 index 0000000000..1fb509b440 --- /dev/null +++ b/addons/mysql/scripts/orc-role-probe.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +role_probe_error() { + printf 'orc role probe failed: %s\n' "$*" >&2 + return 1 +} + +run_orc_role_probe() { + local budget="${ORC_ROLE_PROBE_CLIENT_TIMEOUT_SECONDS:-4}" + if command -v timeout >/dev/null 2>&1; then + timeout "${budget}s" /kubeblocks/orchestrator-client "$@" + return $? + fi + + local temp_dir output_file error_file timeout_file pid timer_pid rc + temp_dir=$(mktemp -d /tmp/orc-role-probe.XXXXXX) || return 1 + output_file="${temp_dir}/output" + error_file="${temp_dir}/error" + timeout_file="${temp_dir}/timeout" + /kubeblocks/orchestrator-client "$@" > "${output_file}" 2> "${error_file}" & + pid=$! + ( + sleep "${budget}" + if kill -0 "${pid}" 2>/dev/null; then + printf 'timeout\n' > "${timeout_file}" + kill "${pid}" 2>/dev/null || true + sleep 1 + kill -9 "${pid}" 2>/dev/null || true + fi + ) & + timer_pid=$! + + wait "${pid}" 2>/dev/null + rc=$? + kill "${timer_pid}" 2>/dev/null || true + wait "${timer_pid}" 2>/dev/null || true + cat "${output_file}" 2>/dev/null || true + cat "${error_file}" >&2 2>/dev/null || true + if [ -s "${timeout_file}" ]; then + rc=124 + fi + rm -rf "${temp_dir}" + return "${rc}" +} + +probe_orchestrator_role() { + local master_info master_from_orc replicas replica rc + + master_info=$(run_orc_role_probe -c which-cluster-master -i "${KB_AGENT_POD_NAME}") + rc=$? + if [ "$rc" -ne 0 ]; then + role_probe_error "cannot determine master (rc=${rc})" + return 1 + fi + if [ -z "$master_info" ]; then + role_probe_error "master query returned empty output" + return 1 + fi + + master_from_orc="${master_info%%:*}" + if [ "$master_from_orc" = "${KB_AGENT_POD_NAME}" ]; then + printf 'primary' + return 0 + fi + + replicas=$(run_orc_role_probe -c which-cluster-instances -i "${master_from_orc}") + rc=$? + if [ "$rc" -ne 0 ]; then + role_probe_error "cannot list replicas for ${master_from_orc} (rc=${rc})" + return 1 + fi + for replica in $replicas; do + if [ "${replica%%:*}" = "${KB_AGENT_POD_NAME}" ]; then + printf 'secondary' + return 0 + fi + done + + role_probe_error "pod ${KB_AGENT_POD_NAME} is absent from Orchestrator topology rooted at ${master_from_orc}" + return 1 +} + +# ShellSpec sets __SOURCED__; load functions without executing the probe. +${__SOURCED__:+false} : || return 0 + +probe_orchestrator_role diff --git a/addons/mysql/scripts/orc-switchover.sh b/addons/mysql/scripts/orc-switchover.sh index ebea5774b1..7c6a6d5e34 100644 --- a/addons/mysql/scripts/orc-switchover.sh +++ b/addons/mysql/scripts/orc-switchover.sh @@ -15,16 +15,383 @@ mysql_error() { exit 1 } +switchover_diagnose_not_ready() { + local phase="$1" + local ctx="$2" + local retry_safe="$3" + { + echo "orc switchover diagnosis:" + echo " action: switchover" + echo " phase: ${phase}" + echo " current: ${KB_SWITCHOVER_CURRENT_NAME:-}" + echo " candidate: ${KB_SWITCHOVER_CANDIDATE_NAME:-}" + echo "${ctx}" + echo " next-retry-safe: ${retry_safe}" + } >&2 +} + +run_command_with_budget() { + local budget="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "${budget}s" "$@" + return $? + fi + + local temp_dir output_file timeout_file pid timer_pid rc + temp_dir=$(mktemp -d /tmp/orc-switchover.XXXXXX) || return 1 + output_file="${temp_dir}/output" + timeout_file="${temp_dir}/timeout" + "$@" > "${output_file}" 2>&1 & + pid=$! + ( + sleep "${budget}" + if kill -0 "${pid}" 2>/dev/null; then + printf 'timeout\n' > "${timeout_file}" + kill "${pid}" 2>/dev/null || true + sleep 1 + kill -9 "${pid}" 2>/dev/null || true + fi + ) & + timer_pid=$! + + wait "${pid}" 2>/dev/null + rc=$? + kill "${timer_pid}" 2>/dev/null || true + wait "${timer_pid}" 2>/dev/null || true + cat "${output_file}" + if [ -s "${timeout_file}" ]; then + rc=124 + fi + rm -rf "${temp_dir}" + return $rc +} + +run_orchestrator_client_with_budget() { + local budget="$1" + shift + run_command_with_budget "${budget}" /kubeblocks/orchestrator-client "$@" +} + +ORC_SWITCHOVER_CLIENT_PID="" +ORC_SWITCHOVER_CLIENT_OUTPUT_FILE="" +ORC_SWITCHOVER_CLIENT_RC_FILE="" +ORC_SWITCHOVER_CLIENT_TEMP_DIR="" +ORC_SWITCHOVER_CLIENT_RC="" +ORC_SWITCHOVER_CLIENT_OUTPUT="" +SWITCHOVER_VERIFY_CANDIDATE_RAW="" +SWITCHOVER_VERIFY_CURRENT_RAW="" + +start_orchestrator_client_background() { + local budget="$1" + shift + if ! ORC_SWITCHOVER_CLIENT_TEMP_DIR=$(mktemp -d /tmp/orc-switchover-client.XXXXXX); then + ORC_SWITCHOVER_CLIENT_RC=1 + ORC_SWITCHOVER_CLIENT_OUTPUT="failed to create orchestrator client temp directory" + return 1 + fi + ORC_SWITCHOVER_CLIENT_OUTPUT_FILE="${ORC_SWITCHOVER_CLIENT_TEMP_DIR}/output" + ORC_SWITCHOVER_CLIENT_RC_FILE="${ORC_SWITCHOVER_CLIENT_TEMP_DIR}/rc" + ORC_SWITCHOVER_CLIENT_RC="" + ORC_SWITCHOVER_CLIENT_OUTPUT="" + ( + run_orchestrator_client_with_budget "${budget}" "$@" > "${ORC_SWITCHOVER_CLIENT_OUTPUT_FILE}" 2>&1 + printf '%s\n' "$?" > "${ORC_SWITCHOVER_CLIENT_RC_FILE}" + ) & + ORC_SWITCHOVER_CLIENT_PID=$! + return 0 +} + +finish_orchestrator_client_background() { + local wrapper_rc=0 + if [ -n "${ORC_SWITCHOVER_CLIENT_PID}" ]; then + wait "${ORC_SWITCHOVER_CLIENT_PID}" + wrapper_rc=$? + fi + + ORC_SWITCHOVER_CLIENT_OUTPUT=$(cat "${ORC_SWITCHOVER_CLIENT_OUTPUT_FILE}" 2>/dev/null || true) + if [ -s "${ORC_SWITCHOVER_CLIENT_RC_FILE}" ]; then + ORC_SWITCHOVER_CLIENT_RC=$(cat "${ORC_SWITCHOVER_CLIENT_RC_FILE}") + else + ORC_SWITCHOVER_CLIENT_RC="${wrapper_rc}" + fi + rm -rf "${ORC_SWITCHOVER_CLIENT_TEMP_DIR}" + ORC_SWITCHOVER_CLIENT_PID="" + ORC_SWITCHOVER_CLIENT_OUTPUT_FILE="" + ORC_SWITCHOVER_CLIENT_RC_FILE="" + ORC_SWITCHOVER_CLIENT_TEMP_DIR="" +} + +mysql_read_flags() { + local host="$1" + local budget="${MYSQL_ORC_SWITCHOVER_MYSQL_TIMEOUT_SECONDS:-1}" + local connect_timeout="${MYSQL_ORC_SWITCHOVER_MYSQL_CONNECT_TIMEOUT_SECONDS:-1}" + run_command_with_budget "${budget}" env MYSQL_PWD="${MYSQL_ROOT_PASSWORD}" mysql -u"${MYSQL_ROOT_USER}" -P3306 -h"${host}" \ + --connect-timeout="${connect_timeout}" --batch --skip-column-names \ + -e "SELECT @@global.read_only, @@global.super_read_only;" 2>&1 +} + +read_mysql_flags_pair() { + local candidate="$1" + local current="$2" + local temp_dir candidate_output_file candidate_rc_file current_output_file current_rc_file + local candidate_pid current_pid candidate_rc current_rc candidate_output current_output + + SWITCHOVER_VERIFY_CANDIDATE_RAW="" + SWITCHOVER_VERIFY_CURRENT_RAW="" + if ! temp_dir=$(mktemp -d /tmp/orc-switchover-readback.XXXXXX); then + SWITCHOVER_VERIFY_CANDIDATE_FLAGS="" + SWITCHOVER_VERIFY_CURRENT_FLAGS="" + return 1 + fi + candidate_output_file="${temp_dir}/candidate-output" + candidate_rc_file="${temp_dir}/candidate-rc" + current_output_file="${temp_dir}/current-output" + current_rc_file="${temp_dir}/current-rc" + + ( + mysql_read_flags "$candidate" > "${candidate_output_file}" 2>&1 + printf '%s\n' "$?" > "${candidate_rc_file}" + ) & + candidate_pid=$! + + ( + mysql_read_flags "$current" > "${current_output_file}" 2>&1 + printf '%s\n' "$?" > "${current_rc_file}" + ) & + current_pid=$! + + wait "${candidate_pid}" 2>/dev/null || true + wait "${current_pid}" 2>/dev/null || true + + candidate_rc=$(cat "${candidate_rc_file}" 2>/dev/null || printf '1') + current_rc=$(cat "${current_rc_file}" 2>/dev/null || printf '1') + candidate_output=$(cat "${candidate_output_file}" 2>/dev/null || true) + current_output=$(cat "${current_output_file}" 2>/dev/null || true) + SWITCHOVER_VERIFY_CANDIDATE_RAW="${candidate_output}" + SWITCHOVER_VERIFY_CURRENT_RAW="${current_output}" + SWITCHOVER_VERIFY_CANDIDATE_FLAGS="rc=${candidate_rc} output=${candidate_output}" + SWITCHOVER_VERIFY_CURRENT_FLAGS="rc=${current_rc} output=${current_output}" + + rm -rf "${temp_dir}" + [ "$candidate_rc" = "0" ] && [ "$current_rc" = "0" ] +} + +is_false_flag() { + case "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" in + 0|off|false) return 0 ;; + *) return 1 ;; + esac +} + +is_true_flag() { + case "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" in + 1|on|true) return 0 ;; + *) return 1 ;; + esac +} + +extract_mysql_read_flags() { + local flags="$1" + printf '%s\n' "$flags" | awk ' + NF >= 2 { + read_only = tolower($1) + super_read_only = tolower($2) + if ((read_only == "0" || read_only == "1" || read_only == "on" || read_only == "off" || read_only == "true" || read_only == "false") && + (super_read_only == "0" || super_read_only == "1" || super_read_only == "on" || super_read_only == "off" || super_read_only == "true" || super_read_only == "false")) { + last_read_only = $1 + last_super_read_only = $2 + found = 1 + } + } + END { + if (!found) { + exit 1 + } + print last_read_only, last_super_read_only + } + ' +} + +is_writable_mysql() { + local flags="$1" + local parsed read_only super_read_only + parsed=$(extract_mysql_read_flags "$flags") || return 1 + read -r read_only super_read_only <<< "$parsed" + is_false_flag "$read_only" && is_false_flag "$super_read_only" +} + +is_readonly_mysql() { + local flags="$1" + local parsed read_only super_read_only + parsed=$(extract_mysql_read_flags "$flags") || return 1 + read -r read_only super_read_only <<< "$parsed" + is_true_flag "$read_only" && is_true_flag "$super_read_only" +} + +append_switchover_verify_history() { + local attempt="$1" + local entry + entry=$(printf 'attempt %s: observed-candidate=%s; candidate-flags=%s; current-flags=%s' \ + "$attempt" "${SWITCHOVER_VERIFY_CANDIDATE:-}" \ + "${SWITCHOVER_VERIFY_CANDIDATE_FLAGS:-}" \ + "${SWITCHOVER_VERIFY_CURRENT_FLAGS:-}") + if [ -z "${SWITCHOVER_VERIFY_HISTORY:-}" ]; then + SWITCHOVER_VERIFY_HISTORY="$entry" + else + SWITCHOVER_VERIFY_HISTORY=$(printf '%s\n%s' "$SWITCHOVER_VERIFY_HISTORY" "$entry") + fi +} + +verify_switchover_closed_once() { + local candidate="${KB_SWITCHOVER_CANDIDATE_NAME:-}" + local master_from_orc precheck_budget rc + precheck_budget="${MYSQL_ORC_SWITCHOVER_PRECHECK_TIMEOUT_SECONDS:-3}" + + if [ -z "$candidate" ]; then + master_from_orc=$(run_orchestrator_client_with_budget "${precheck_budget}" -c which-cluster-master -i "${KB_SWITCHOVER_CURRENT_NAME}" 2>&1) + rc=$? + if [ $rc -ne 0 ]; then + SWITCHOVER_VERIFY_CANDIDATE_FLAGS="candidate lookup rc=${rc} output=${master_from_orc}" + SWITCHOVER_VERIFY_CURRENT_FLAGS="" + return 1 + fi + candidate="${master_from_orc%%:*}" + fi + SWITCHOVER_VERIFY_CANDIDATE="${candidate:-}" + + if [ -z "$candidate" ] || [ "$candidate" = "$KB_SWITCHOVER_CURRENT_NAME" ]; then + SWITCHOVER_VERIFY_CANDIDATE_FLAGS="" + SWITCHOVER_VERIFY_CURRENT_FLAGS="" + return 1 + fi + + if ! read_mysql_flags_pair "$candidate" "$KB_SWITCHOVER_CURRENT_NAME"; then + return 1 + fi + + is_writable_mysql "$SWITCHOVER_VERIFY_CANDIDATE_RAW" && is_readonly_mysql "$SWITCHOVER_VERIFY_CURRENT_RAW" +} + +switchover_verify_context() { + local attempts="${MYSQL_ORC_SWITCHOVER_VERIFY_ATTEMPTS:-20}" + local interval="${MYSQL_ORC_SWITCHOVER_VERIFY_INTERVAL_SECONDS:-1}" + local window="${MYSQL_ORC_SWITCHOVER_VERIFY_WINDOW_SECONDS:-${MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS:-40}}" + local ctx + ctx=$(printf ' verify-attempts: %s\n verify-interval-seconds: %s\n verify-window-seconds: %s\n precheck-timeout-seconds: %s\n client-timeout-seconds: %s\n mysql-timeout-seconds: %s\n mysql-connect-timeout-seconds: %s\n observed-candidate: %s\n candidate-flags: %s\n current-flags: %s' \ + "$attempts" "$interval" "$window" "${MYSQL_ORC_SWITCHOVER_PRECHECK_TIMEOUT_SECONDS:-3}" \ + "${MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS:-40}" \ + "${MYSQL_ORC_SWITCHOVER_MYSQL_TIMEOUT_SECONDS:-1}" \ + "${MYSQL_ORC_SWITCHOVER_MYSQL_CONNECT_TIMEOUT_SECONDS:-1}" \ + "${SWITCHOVER_VERIFY_CANDIDATE:-}" \ + "${SWITCHOVER_VERIFY_CANDIDATE_FLAGS:-}" "${SWITCHOVER_VERIFY_CURRENT_FLAGS:-}") + printf '%s\n verify-history:\n%s' "$ctx" "${SWITCHOVER_VERIFY_HISTORY:-}" +} + +diagnose_switchover_not_converged() { + local extra="${1:-}" + local ctx + ctx=$(switchover_verify_context) + if [ -n "$extra" ]; then + ctx=$(printf '%s\n%s' "$ctx" "$extra") + fi + switchover_diagnose_not_ready "post-switchover-not-converged" "$ctx" "yes" +} + +verify_switchover_closed_window() { + local attempts="${MYSQL_ORC_SWITCHOVER_VERIFY_ATTEMPTS:-20}" + local interval="${MYSQL_ORC_SWITCHOVER_VERIFY_INTERVAL_SECONDS:-1}" + local window="${MYSQL_ORC_SWITCHOVER_VERIFY_WINDOW_SECONDS:-${MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS:-40}}" + local started="$SECONDS" + local i + + SWITCHOVER_VERIFY_HISTORY="" + i=1 + while [ "$i" -le "$attempts" ]; do + if [ "$i" -gt 1 ] && [ $((SECONDS - started)) -ge "$window" ]; then + break + fi + if verify_switchover_closed_once; then + mysql_note "Switchover verified: candidate is writable and previous primary is read-only." + return 0 + fi + append_switchover_verify_history "$i" + if [ "$i" -lt "$attempts" ] && [ $((SECONDS - started)) -lt "$window" ]; then + sleep "$interval" + fi + i=$((i + 1)) + done + + return 1 +} + +verify_switchover_closed_or_defer() { + if verify_switchover_closed_window; then + return 0 + fi + diagnose_switchover_not_converged + return 1 +} + +orchestrator_client_context() { + printf ' orchestrator-client-rc: %s\n orchestrator-client-output:\n%s' \ + "${ORC_SWITCHOVER_CLIENT_RC:-}" "${ORC_SWITCHOVER_CLIENT_OUTPUT:-}" +} + +run_switchover_client_and_verify() { + local client_budget="$1" + local verify_rc + shift + + if ! start_orchestrator_client_background "${client_budget}" "$@"; then + switchover_diagnose_not_ready "orchestrator-client-start-failed" \ + "$(orchestrator_client_context)" "yes" + return 1 + fi + if verify_switchover_closed_window; then + verify_rc=0 + else + verify_rc=1 + fi + finish_orchestrator_client_background + + if [ "$verify_rc" -eq 0 ]; then + if [ "${ORC_SWITCHOVER_CLIENT_RC:-0}" != "0" ]; then + mysql_note "Switchover command returned non-zero (${ORC_SWITCHOVER_CLIENT_RC}) but post-check observed the target topology." + if [ -n "${ORC_SWITCHOVER_CLIENT_OUTPUT:-}" ]; then + mysql_note "${ORC_SWITCHOVER_CLIENT_OUTPUT}" + fi + fi + return 0 + fi + + diagnose_switchover_not_converged "$(orchestrator_client_context)" + if [ "${ORC_SWITCHOVER_CLIENT_RC:-0}" != "0" ]; then + switchover_diagnose_not_ready "orchestrator-command-failed" \ + "$(printf ' rc: %s\n observed:\n%s' "${ORC_SWITCHOVER_CLIENT_RC}" "${ORC_SWITCHOVER_CLIENT_OUTPUT:-}")" "yes" + fi + return 1 +} + +# This is magic for shellspec ut framework. +# When included from shellspec, __SOURCED__ is set and only functions are loaded. +${__SOURCED__:+false} : || return 0 + # Check pod role if [[ "$KB_SWITCHOVER_ROLE" != "primary" ]]; then mysql_note "Switchover not triggered for non-primary role, skipping." exit 0 fi -# Skip if KB_SWITCHOVER_CURRENT_NAME is not the master -master_from_orc=$(/kubeblocks/orchestrator-client -c which-cluster-master -i "${KB_SWITCHOVER_CURRENT_NAME}" 2>&1) -if [ -z "$master_from_orc" ]; then - mysql_error "Could not determine current master from Orchestrator" +# Skip if KB_SWITCHOVER_CURRENT_NAME is not the master. +# Keep Orchestrator probing bounded; rc!=0 is a hard precheck failure and must +# not be mistaken for a master name (that previously made this guard exit 0). +precheck_budget="${MYSQL_ORC_SWITCHOVER_PRECHECK_TIMEOUT_SECONDS:-3}" + +master_from_orc=$(run_orchestrator_client_with_budget "${precheck_budget}" -c which-cluster-master -i "${KB_SWITCHOVER_CURRENT_NAME}" 2>&1) +rc=$? +if [ $rc -ne 0 ] || [ -z "$master_from_orc" ]; then + mysql_error "Could not determine current master from Orchestrator (rc=${rc}): ${master_from_orc}" fi if [ "${KB_SWITCHOVER_CURRENT_NAME}" != "${master_from_orc%%:*}" ]; then @@ -33,8 +400,13 @@ if [ "${KB_SWITCHOVER_CURRENT_NAME}" != "${master_from_orc%%:*}" ]; then fi # Skip switch if there is only one instance -instance_count=$(/kubeblocks/orchestrator-client -c which-cluster-instances -i "${KB_SWITCHOVER_CURRENT_NAME}" 2>&1 | wc -l) -if [ "$instance_count" -eq 1 ]; then +instances=$(run_orchestrator_client_with_budget "${precheck_budget}" -c which-cluster-instances -i "${KB_SWITCHOVER_CURRENT_NAME}" 2>&1) +rc=$? +if [ $rc -ne 0 ]; then + mysql_error "Could not list cluster instances from Orchestrator (rc=${rc}): ${instances}" +fi +instance_count=$(printf '%s\n' "$instances" | sed '/^$/d' | wc -l) +if [ "$instance_count" -le 1 ]; then mysql_note "Only one instance in cluster, cannot switchover." exit 0 fi @@ -42,18 +414,18 @@ fi if [ -n "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then # Switchover to specific candidate mysql_note "Initiating graceful switchover to: ${KB_SWITCHOVER_CANDIDATE_NAME}" - result=$(/kubeblocks/orchestrator-client -c graceful-master-takeover-auto \ + if run_switchover_client_and_verify "${MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS:-40}" -c graceful-master-takeover-auto \ -i "${KB_SWITCHOVER_CURRENT_NAME}" \ - -d "${KB_SWITCHOVER_CANDIDATE_NAME}" 2>&1) - exit_code=$? + -d "${KB_SWITCHOVER_CANDIDATE_NAME}"; then + exit 0 + fi else # Auto-select candidate mysql_note "Initiating graceful switchover with auto-selected candidate" - result=$(/kubeblocks/orchestrator-client -c graceful-master-takeover-auto \ - -i "${KB_SWITCHOVER_CURRENT_NAME}" 2>&1) - exit_code=$? + if run_switchover_client_and_verify "${MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS:-40}" -c graceful-master-takeover-auto \ + -i "${KB_SWITCHOVER_CURRENT_NAME}"; then + exit 0 + fi fi -if [ $exit_code -ne 0 ]; then - mysql_error "Switchover command failed with exit code: ${result}" -fi \ No newline at end of file +exit 1 diff --git a/addons/mysql/templates/_orc.tpl b/addons/mysql/templates/_orc.tpl index e788685ae8..88079cd990 100644 --- a/addons/mysql/templates/_orc.tpl +++ b/addons/mysql/templates/_orc.tpl @@ -184,36 +184,19 @@ accountProvision: targetPodSelector: Role matchingKey: primary roleProbe: - periodSeconds: {{ .Values.roleProbe.periodSeconds }} - timeoutSeconds: {{ .Values.roleProbe.timeoutSeconds }} + periodSeconds: {{ .Values.roleProbe.orc.periodSeconds }} + timeoutSeconds: {{ .Values.roleProbe.orc.timeoutSeconds }} exec: env: - name: PATH value: /kubeblocks/:/kubeblocks-tools/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: ORC_ROLE_PROBE_CLIENT_TIMEOUT_SECONDS + value: "{{ .Values.roleProbe.orc.clientTimeoutSeconds }}" command: - /bin/bash - -c - | - master_info=$(/kubeblocks/orchestrator-client -c which-cluster-master -i "${KB_AGENT_POD_NAME}") || true - if [[ -z "$master_info" ]]; then - echo -n "" - exit 0 - fi - master_from_orc="${master_info%%:*}" - if [ "$master_from_orc" == "${KB_AGENT_POD_NAME}" ]; then - echo -n "primary" - else - # get list of replicas - replicas=$(/kubeblocks/orchestrator-client -c which-cluster-instances -i "${master_from_orc}") - # for each replica, check if it is a secondary - for replica in $replicas; do - if [ "${replica%%:*}" == "${KB_AGENT_POD_NAME}" ]; then - echo -n "secondary" - else - echo -n "" - fi - done - fi + exec /orc-scripts/role-probe.sh memberLeave: exec: command: @@ -226,15 +209,36 @@ memberLeave: exit 1 fi switchover: + timeoutSeconds: {{ .Values.switchover.timeoutSeconds }} exec: + env: + - name: MYSQL_ORC_SWITCHOVER_PRECHECK_TIMEOUT_SECONDS + value: "{{ .Values.switchover.precheckTimeoutSeconds }}" + - name: MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS + value: "{{ .Values.switchover.clientTimeoutSeconds }}" + - name: MYSQL_ORC_SWITCHOVER_MYSQL_TIMEOUT_SECONDS + value: "{{ .Values.switchover.mysqlTimeoutSeconds }}" + - name: MYSQL_ORC_SWITCHOVER_MYSQL_CONNECT_TIMEOUT_SECONDS + value: "{{ .Values.switchover.mysqlConnectTimeoutSeconds }}" + - name: MYSQL_ORC_SWITCHOVER_VERIFY_ATTEMPTS + value: "{{ .Values.switchover.verifyAttempts }}" + - name: MYSQL_ORC_SWITCHOVER_VERIFY_INTERVAL_SECONDS + value: "{{ .Values.switchover.verifyIntervalSeconds }}" command: - - /bin/sh + - /bin/bash - -c - | - /orc-scripts/switchover.sh 2>> /tmp/switchover.log - if [ $? -ne 0 ]; then - echo "ERROR: Failed to switchover" - exit 1 + /orc-scripts/switchover.sh 2> >(tee -a /tmp/switchover.log >&2) + rc=$? + if [ $rc -ne 0 ]; then + { + echo "ERROR: Failed to switchover (rc=${rc}); diagnostics mirrored to /tmp/switchover.log" + if [ -s /tmp/switchover.log ]; then + echo "ERROR: switchover diagnostic tail:" + tail -n 200 /tmp/switchover.log || true + fi + } | tee /dev/stderr + exit $rc fi {{- end }} diff --git a/addons/mysql/templates/scripts.yaml b/addons/mysql/templates/scripts.yaml index 2a85068ad6..71d1d4c7d8 100644 --- a/addons/mysql/templates/scripts.yaml +++ b/addons/mysql/templates/scripts.yaml @@ -45,6 +45,8 @@ metadata: labels: {{- include "mysql.labels" . | nindent 4 }} data: + role-probe.sh: |- + {{- .Files.Get "scripts/orc-role-probe.sh" | nindent 4 }} switchover.sh: |- {{- .Files.Get "scripts/orc-switchover.sh" | nindent 4 }} member-leave.sh: |- diff --git a/addons/mysql/values.yaml b/addons/mysql/values.yaml index be3d70baed..d0d243ca69 100644 --- a/addons/mysql/values.yaml +++ b/addons/mysql/values.yaml @@ -117,7 +117,22 @@ roleProbe: failureThreshold: 2 periodSeconds: 1 timeoutSeconds: 1 + orc: + periodSeconds: 5 + timeoutSeconds: 5 + clientTimeoutSeconds: 4 + +switchover: + # kbagent caps positive action timeouts at 60s; keep a buffer for diagnostics. + # Worst-case budget: 2*3s prechecks + max(40s ORC CLI, 20*(parallel 1s SQL probes) + 19*1s intervals) = 46s. + timeoutSeconds: 50 + precheckTimeoutSeconds: 3 + clientTimeoutSeconds: 40 + mysqlTimeoutSeconds: 1 + mysqlConnectTimeoutSeconds: 1 + verifyAttempts: 20 + verifyIntervalSeconds: 1 ## @param cmpdNamePrefix for each ComponentDefinition resources name created by this chart, that can avoid name conflict ## If specified, the component definition will use it as prefix. -cmpdNamePrefix: "mysql" \ No newline at end of file +cmpdNamePrefix: "mysql"