Skip to content

Commit 82e1ab7

Browse files
committed
fix(cluster): make raft readiness timeout configurable via raft_ready_timeout_ms
B3 (2026-08-26): RAFT_READY_TIMEOUT=30s hardcoded vs replay of a large raft log (>150k entries) exceeding 30s — ReadyGate Failed, process alive but port dead. New [server] raft_ready_timeout_ms config key (default 30_000 keeps existing behaviour).
1 parent f3b6838 commit 82e1ab7

3 files changed

Lines changed: 15 additions & 4 deletions

File tree

‎nodedb/src/bootstrap/cluster_ready.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ pub async fn await_cluster_ready(
3333
raft_ready_rx: Option<tokio::sync::watch::Receiver<bool>>,
3434
data_plane_replay_done: Vec<tokio::sync::oneshot::Receiver<()>>,
3535
gates: ClusterReadyGates,
36+
raft_ready_timeout: std::time::Duration,
3637
) -> anyhow::Result<()> {
3738
let ClusterReadyGates {
3839
raft_gate,
@@ -51,8 +52,7 @@ pub async fn await_cluster_ready(
5152
// `metadata propose: not leader` because election had not yet
5253
// completed.
5354
if let Some(mut ready_rx) = raft_ready_rx {
54-
const RAFT_READY_TIMEOUT: Duration = Duration::from_secs(30);
55-
match tokio::time::timeout(RAFT_READY_TIMEOUT, ready_rx.wait_for(|v| *v)).await {
55+
match tokio::time::timeout(raft_ready_timeout, ready_rx.wait_for(|v| *v)).await {
5656
Ok(Ok(_)) => {
5757
info!("metadata raft group ready — opening client listeners");
5858
}
@@ -64,10 +64,10 @@ pub async fn await_cluster_ready(
6464
}
6565
Err(_) => {
6666
raft_gate.fail(format!(
67-
"raft readiness timeout after {RAFT_READY_TIMEOUT:?}"
67+
"raft readiness timeout after {raft_ready_timeout:?}"
6868
));
6969
return Err(anyhow::anyhow!(
70-
"raft readiness timeout after {RAFT_READY_TIMEOUT:?} — \
70+
"raft readiness timeout after {raft_ready_timeout:?} — \
7171
metadata group failed to apply first entry"
7272
));
7373
}

‎nodedb/src/config/server/section.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,13 @@ pub struct ServerSection {
7171
#[serde(default)]
7272
pub log_format: LogFormat,
7373

74+
/// Boot-time raft metadata group readiness timeout in milliseconds.
75+
/// Default 30_000 (30s). Replay of a large raft log (long write bursts,
76+
/// big migrations) can exceed 30s — raise this via `raft_ready_timeout_ms`.
77+
/// Script/config set: `[server] raft_ready_timeout_ms = 300000`.
78+
#[serde(default)]
79+
pub raft_ready_timeout_ms: Option<u64>,
80+
7481
/// Per-listener TLS configuration. Lives under `[server.tls]` because
7582
/// it directly modifies the listeners declared in `[server.ports]`
7683
/// (per-protocol toggles + cert paths). When absent, every listener
@@ -126,6 +133,7 @@ impl Default for ServerSection {
126133
memory_limit: default_memory_limit(),
127134
max_connections: default_max_connections(),
128135
log_format: LogFormat::Text,
136+
raft_ready_timeout_ms: None,
129137
tls: None,
130138
single_node_calvin: default_single_node_calvin(),
131139
native_crash_dumps: false,

‎nodedb/src/main.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,9 @@ async fn server_main() -> anyhow::Result<()> {
243243
health_loop_gate,
244244
gateway_enable_gate,
245245
},
246+
std::time::Duration::from_millis(
247+
config.server.raft_ready_timeout_ms.unwrap_or(30_000),
248+
),
246249
)
247250
.await?;
248251

0 commit comments

Comments
 (0)