fix(mysql): bound ORC roleProbe client calls

addons/mysql/scripts-ut-spec/orc_role_probe_spec.sh

@@ -0,0 +1,107 @@
+# shellcheck shell=bash
+# shellcheck disable=SC2034,SC2329
+
+Describe "ORC role probe script tests"
+  Include ../scripts/orc-role-probe.sh
+
+  setup_role_probe() {
+    export KB_AGENT_POD_NAME="mysql-0"
+  }
+
+  cleanup_role_probe() {
+    unset KB_AGENT_POD_NAME
+  }
+
+  Before 'setup_role_probe'
+  After 'cleanup_role_probe'
+
+  It "publishes primary when Orchestrator identifies the local pod as master"
+    run_orc_role_probe() {
+      printf 'mysql-0:3306\n'
+    }
+
+    When call probe_orchestrator_role
+    The status should be success
+    The output should equal "primary"
+  End
+
+  It "does not parse successful client stderr noise as the master name"
+    run_orc_role_probe() {
+      printf 'transient client warning\n' >&2
+      printf 'mysql-0:3306\n'
+    }
+
+    When call probe_orchestrator_role
+    The status should be success
+    The output should equal "primary"
+    The error should include "transient client warning"
+  End
+
+  It "publishes secondary when the local pod is in the replica list"
+    run_orc_role_probe() {
+      if [ "$2" = "which-cluster-master" ]; then
+        printf 'mysql-1:3306\n'
+      else
+        printf 'mysql-0:3306\nmysql-2:3306\n'
+      fi
+    }
+
+    When call probe_orchestrator_role
+    The status should be success
+    The output should equal "secondary"
+  End
+
+  It "fails instead of publishing an empty role when the master query fails"
+    run_orc_role_probe() {
+      printf 'orchestrator unavailable\n'
+      return 1
+    }
+
+    When call probe_orchestrator_role
+    The status should be failure
+    The error should include "cannot determine master"
+    The output should equal ""
+  End
+
+  It "fails instead of publishing an empty role when the master output is empty"
+    run_orc_role_probe() {
+      return 0
+    }
+
+    When call probe_orchestrator_role
+    The status should be failure
+    The error should include "master query returned empty output"
+    The output should equal ""
+  End
+
+  It "fails instead of publishing an empty role when the replica query fails"
+    run_orc_role_probe() {
+      if [ "$2" = "which-cluster-master" ]; then
+        printf 'mysql-1:3306\n'
+        return 0
+      fi
+      printf 'replica query failed\n'
+      return 1
+    }
+
+    When call probe_orchestrator_role
+    The status should be failure
+    The error should include "cannot list replicas"
+    The output should equal ""
+  End
+
+  It "fails instead of publishing an empty role when the pod is absent from topology"
+    run_orc_role_probe() {
+      if [ "$2" = "which-cluster-master" ]; then
+        printf 'mysql-1:3306\n'
+      else
+        printf 'mysql-2:3306\n'
+      fi
+    }
+
+    When call probe_orchestrator_role
+    The status should be failure
+    The error should include "is absent from Orchestrator topology"
+    The output should equal ""
+  End
+End

addons/mysql/scripts-ut-spec/orc_switchover_spec.sh

@@ -0,0 +1,197 @@
+# shellcheck shell=bash
+# shellcheck disable=SC2034,SC2329
+
+Describe "ORC switchover script tests"
+  Include ../scripts/orc-switchover.sh
+
+  Describe "MySQL read flag parsing"
+    It "recognizes writable flags after mysql client stderr noise"
+      output=$(printf '%s\n%s\n' \
+        'mysql: [Warning] Using a password on the command line interface can be insecure.' \
+        '0	0')
+
+      When call is_writable_mysql "$output"
+      The status should be success
+    End
+
+    It "recognizes read-only flags after mysql client stderr noise"
+      output=$(printf '%s\n%s\n' \
+        'mysql: [Warning] Using a password on the command line interface can be insecure.' \
+        '1	1')
+
+      When call is_readonly_mysql "$output"
+      The status should be success
+    End
+
+    It "rejects output without a read_only/super_read_only row"
+      output='mysql: [Warning] Using a password on the command line interface can be insecure.'
+
+      When call is_writable_mysql "$output"
+      The status should be failure
+    End
+  End
+
+  Describe "Switchover closure verification"
+    setup_switchover_verify() {
+      export KB_SWITCHOVER_CURRENT_NAME="mysql-0"
+      export KB_SWITCHOVER_CANDIDATE_NAME="mysql-1"
+      export MYSQL_ORC_SWITCHOVER_VERIFY_ATTEMPTS=20
+      export MYSQL_ORC_SWITCHOVER_VERIFY_INTERVAL_SECONDS=0
+      export MYSQL_ORC_SWITCHOVER_PRECHECK_TIMEOUT_SECONDS=3
+      export MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS=40
+      export MYSQL_ORC_SWITCHOVER_MYSQL_TIMEOUT_SECONDS=1
+      export MYSQL_ORC_SWITCHOVER_MYSQL_CONNECT_TIMEOUT_SECONDS=1
+      VERIFY_COUNTER_FILE=$(mktemp)
+      export VERIFY_COUNTER_FILE
+      printf '0\n' > "$VERIFY_COUNTER_FILE"
+    }
+
+    cleanup_switchover_verify() {
+      rm -f "${VERIFY_COUNTER_FILE:-}"
+      unset KB_SWITCHOVER_CURRENT_NAME
+      unset KB_SWITCHOVER_CANDIDATE_NAME
+      unset MYSQL_ORC_SWITCHOVER_VERIFY_ATTEMPTS
+      unset MYSQL_ORC_SWITCHOVER_VERIFY_INTERVAL_SECONDS
+      unset MYSQL_ORC_SWITCHOVER_PRECHECK_TIMEOUT_SECONDS
+      unset MYSQL_ORC_SWITCHOVER_CLIENT_TIMEOUT_SECONDS
+      unset MYSQL_ORC_SWITCHOVER_MYSQL_TIMEOUT_SECONDS
+      unset MYSQL_ORC_SWITCHOVER_MYSQL_CONNECT_TIMEOUT_SECONDS
+      unset VERIFY_COUNTER_FILE
+      unset ORC_SWITCHOVER_CLIENT_PID
+      unset ORC_SWITCHOVER_CLIENT_OUTPUT_FILE
+      unset ORC_SWITCHOVER_CLIENT_RC_FILE
+      unset ORC_SWITCHOVER_CLIENT_TEMP_DIR
+      unset ORC_SWITCHOVER_CLIENT_RC
+      unset ORC_SWITCHOVER_CLIENT_OUTPUT
+    }
+
+    Before 'setup_switchover_verify'
+    After 'cleanup_switchover_verify'
+
+    It "succeeds when readback converges inside the bounded verify window"
+      mysql_read_flags() {
+        local host="$1"
+        local count
+        if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then
+          count=$(cat "$VERIFY_COUNTER_FILE")
+          count=$((count + 1))
+          printf '%s\n' "$count" > "$VERIFY_COUNTER_FILE"
+          if [ "$count" -lt 3 ]; then
+            printf '1 1\n'
+            return 0
+          fi
+          printf '0 0\n'
+          return 0
+        fi
+        printf '1 1\n'
+      }
+
+      When call verify_switchover_closed_or_defer
+      The status should be success
+      The output should include "Switchover verified"
+    End
+
+    It "uses raw parallel readback output for closure checks"
+      mysql_read_flags() {
+        local host="$1"
+        printf '%s\n' 'mysql: [Warning] Using a password on the command line interface can be insecure.'
+        if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then
+          printf '0 0\n'
+          return 0
+        fi
+        printf '1 1\n'
+      }
+
+      When call verify_switchover_closed_once
+      The status should be success
+    End
+
+    It "classifies an unclosed readback window as retry-safe"
+      mysql_read_flags() {
+        local host="$1"
+        if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then
+          printf '1 1\n'
+          return 0
+        fi
+        printf '1 1\n'
+      }
+
+      When call verify_switchover_closed_or_defer
+      The status should be failure
+      The error should include "phase: post-switchover-not-converged"
+      The error should include "next-retry-safe: yes"
+      The error should include "verify-history:"
+    End
+
+    It "accepts a non-zero orchestrator client result when same invocation verifies closure"
+      run_orchestrator_client_with_budget() {
+        printf 'client timed out\n'
+        return 124
+      }
+
+      mysql_read_flags() {
+        local host="$1"
+        if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then
+          printf '0 0\n'
+          return 0
+        fi
+        printf '1 1\n'
+      }
+
+      When call run_switchover_client_and_verify 40 -c graceful-master-takeover-auto -i "$KB_SWITCHOVER_CURRENT_NAME" -d "$KB_SWITCHOVER_CANDIDATE_NAME"
+      The status should be success
+      The output should include "Switchover command returned non-zero (124) but post-check observed the target topology."
+      The output should include "client timed out"
+    End
+
+    It "keeps unclosed readback retry-safe with orchestrator client diagnostics"
+      run_orchestrator_client_with_budget() {
+        printf 'client timed out\n'
+        return 124
+      }
+
+      mysql_read_flags() {
+        local host="$1"
+        if [ "$host" = "$KB_SWITCHOVER_CANDIDATE_NAME" ]; then
+          printf '1 1\n'
+          return 0
+        fi
+        printf '1 1\n'
+      }
+
+      When call run_switchover_client_and_verify 40 -c graceful-master-takeover-auto -i "$KB_SWITCHOVER_CURRENT_NAME" -d "$KB_SWITCHOVER_CANDIDATE_NAME"
+      The status should be failure
+      The error should include "phase: post-switchover-not-converged"
+      The error should include "orchestrator-client-rc: 124"
+      The error should include "client timed out"
+      The error should include "phase: orchestrator-command-failed"
+    End
+
+    It "fails explicitly when the orchestrator client background wrapper cannot start"
+      start_orchestrator_client_background() {
+        ORC_SWITCHOVER_CLIENT_RC=1
+        ORC_SWITCHOVER_CLIENT_OUTPUT="failed to create orchestrator client temp directory"
+        return 1
+      }
+
+      When call run_switchover_client_and_verify 40 -c graceful-master-takeover-auto -i "$KB_SWITCHOVER_CURRENT_NAME" -d "$KB_SWITCHOVER_CANDIDATE_NAME"
+      The status should be failure
+      The error should include "phase: orchestrator-client-start-failed"
+      The error should include "failed to create orchestrator client temp directory"
+    End
+
+    It "does not reuse stale readback flags when its temp directory cannot be created"
+      SWITCHOVER_VERIFY_CANDIDATE_RAW="0 0"
+      SWITCHOVER_VERIFY_CURRENT_RAW="1 1"
+      mktemp() {
+        return 1
+      }
+
+      When call read_mysql_flags_pair "$KB_SWITCHOVER_CANDIDATE_NAME" "$KB_SWITCHOVER_CURRENT_NAME"
+      The status should be failure
+      The variable SWITCHOVER_VERIFY_CANDIDATE_RAW should equal ""
+      The variable SWITCHOVER_VERIFY_CURRENT_RAW should equal ""
+      The variable SWITCHOVER_VERIFY_CANDIDATE_FLAGS should include "failed to create readback temp directory"
+    End
+  End
+End

addons/mysql/scripts/orc-role-probe.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+role_probe_error() {
+  printf 'orc role probe failed: %s\n' "$*" >&2
+  return 1
+}
+
+run_orc_role_probe() {
+  local budget="${ORC_ROLE_PROBE_CLIENT_TIMEOUT_SECONDS:-4}"
+  if command -v timeout >/dev/null 2>&1; then
+    timeout "${budget}s" /kubeblocks/orchestrator-client "$@"
+    return $?
+  fi
+
+  local temp_dir output_file error_file timeout_file pid timer_pid rc
+  temp_dir=$(mktemp -d /tmp/orc-role-probe.XXXXXX) || return 1
+  output_file="${temp_dir}/output"
+  error_file="${temp_dir}/error"
+  timeout_file="${temp_dir}/timeout"
+  /kubeblocks/orchestrator-client "$@" > "${output_file}" 2> "${error_file}" &
+  pid=$!
+  (
+    sleep "${budget}"
+    if kill -0 "${pid}" 2>/dev/null; then
+      printf 'timeout\n' > "${timeout_file}"
+      kill "${pid}" 2>/dev/null || true
+      sleep 1
+      kill -9 "${pid}" 2>/dev/null || true
+    fi
+  ) &
+  timer_pid=$!
+
+  wait "${pid}" 2>/dev/null
+  rc=$?
+  kill "${timer_pid}" 2>/dev/null || true
+  wait "${timer_pid}" 2>/dev/null || true
+  cat "${output_file}" 2>/dev/null || true
+  cat "${error_file}" >&2 2>/dev/null || true
+  if [ -s "${timeout_file}" ]; then
+    rc=124
+  fi
+  rm -rf "${temp_dir}"
+  return "${rc}"
+}
+
+probe_orchestrator_role() {
+  local master_info master_from_orc replicas replica rc
+
+  master_info=$(run_orc_role_probe -c which-cluster-master -i "${KB_AGENT_POD_NAME}")
+  rc=$?
+  if [ "$rc" -ne 0 ]; then
+    role_probe_error "cannot determine master (rc=${rc})"
+    return 1
+  fi
+  if [ -z "$master_info" ]; then
+    role_probe_error "master query returned empty output"
+    return 1
+  fi
+
+  master_from_orc="${master_info%%:*}"
+  if [ "$master_from_orc" = "${KB_AGENT_POD_NAME}" ]; then
+    printf 'primary'
+    return 0
+  fi
+
+  replicas=$(run_orc_role_probe -c which-cluster-instances -i "${master_from_orc}")
+  rc=$?
+  if [ "$rc" -ne 0 ]; then
+    role_probe_error "cannot list replicas for ${master_from_orc} (rc=${rc})"
+    return 1
+  fi
+  for replica in $replicas; do
+    if [ "${replica%%:*}" = "${KB_AGENT_POD_NAME}" ]; then
+      printf 'secondary'
+      return 0
+    fi
+  done
+
+  role_probe_error "pod ${KB_AGENT_POD_NAME} is absent from Orchestrator topology rooted at ${master_from_orc}"
+  return 1
+}
+
+# ShellSpec sets __SOURCED__; load functions without executing the probe.
+${__SOURCED__:+false} : || return 0
+
+probe_orchestrator_role