diff --git a/.gitignore b/.gitignore index bea80fb..7a2b6b0 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ archived # Generated by deploy/cluster/tokens.sh. A credential that reaches a repository is not a # credential. deploy/cluster/secrets +deploy/k8s/secrets deploy/single/secrets # `make start`: the local database, its log and its pid. A playground, not a fixture. diff --git a/README.md b/README.md index e3c0493..0627071 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ crates/big-bin/ the `big` and `bigctl` binaries crates/big-http/ the server crates/big-embed/ the library API contrib/ written against the HTTP surface, as an outside user would -deploy/ single-node and cluster deployments +deploy/ single-node, compose cluster and Kubernetes deployments examples/ runnable docker compose stacks bench/ benchmarks; excluded from the workspace test run fuzz/ cargo-fuzz targets, nightly only diff --git a/crates/big-bin/src/bin/big.rs b/crates/big-bin/src/bin/big.rs index 9e3818d..14e50c9 100644 --- a/crates/big-bin/src/bin/big.rs +++ b/crates/big-bin/src/bin/big.rs @@ -51,6 +51,10 @@ Offline, on a file nothing else has open: scrub recompute every checksum can reach opening checks the meta page and the chains; this checks the trees, which nothing on the query path ever does + leaks account for every page: reachable, free, or neither + a page in neither is one nothing will ever read and nothing + will ever reuse; one at the end of the file stops `compact` + giving anything back. reads only, and never repairs A backup is an ordinary database file. Restoring is opening it - `restore` exists so that the procedure has a name, not because the file needs converting. diff --git a/crates/big-bin/src/offline.rs b/crates/big-bin/src/offline.rs index d403dc5..1556ffd 100644 --- a/crates/big-bin/src/offline.rs +++ b/crates/big-bin/src/offline.rs @@ -33,6 +33,7 @@ pub fn main(args: &[String], usage: &str) { ["compact", file] => compact(file), ["verify", file] => verify(file), ["scrub", file] => scrub(file), + ["leaks", file] => leaks(file), ["drop-days", file, table, field, before] => drop_days(file, table, field, before), // Named rather than answered with the usage alone: an operator who typed a real // command with the wrong arguments has made a different mistake from one who typed a @@ -62,7 +63,8 @@ pub fn main(args: &[String], usage: &str) { /// Every command word `big` answers to, for telling a typo from a misuse. `serve` is here /// because it is a real command of this binary, just not one this half handles - so /// `big serve` with no file reaches its own error rather than "no such command". -const KNOWN: [&str; 7] = ["serve", "backup", "restore", "compact", "verify", "drop-days", "scrub"]; +const KNOWN: [&str; 8] = + ["serve", "backup", "restore", "compact", "verify", "drop-days", "scrub", "leaks"]; fn backup(file: &str, dest: &str) -> Result<(), String> { let db = open(file)?; @@ -147,6 +149,70 @@ fn scrub(file: &str) -> Result<(), String> { Ok(()) } +/// Accounts for every page in the file: reachable, free, or neither. +/// +/// `verify` already prints `page_count` and the free counters, but nothing ties them to each +/// other or to the size of the file. This does: it walks every tree the current roots and every +/// snapshot can reach, adds the meta pages, the four chains and the freelist, and reports what +/// is left over. **A page in neither set is one the database will never read and never hand out +/// again**, and one of those at the end of the file is enough to stop `compact` giving anything +/// back to the filesystem. +/// +/// A separate subcommand for the same reason as `scrub`: it costs a full read of the live data, +/// while `verify` costs almost nothing. It only reads - there is no repair here, and the counts +/// it reports are exact as of the transaction it opened at rather than of any later one. +/// +/// This is not a scrub. The walk parses pages without verifying their checksums, so run +/// `big scrub ` first if the file is under suspicion. +fn leaks(file: &str) -> Result<(), String> { + let db = open(file)?; + let r = db.audit_pages().map_err(|e| format!("{file} could not be audited: {e}"))?; + + println!("{file}: txn {}, {} pages", r.txn_id, r.page_count); + println!( + " {} reachable ({} meta, {} chain, {} tree, {} snapshot chain, {} snapshot tree)", + r.reachable, + r.by_class.meta, + r.by_class.chains, + r.by_class.trees, + r.by_class.snapshot_chains, + r.by_class.snapshot_trees + ); + println!( + " {} free ({} reusable now, {} still pending)", + r.free_total, + r.free_reusable, + r.free_total - r.free_reusable + ); + println!(" {} unaccounted for", r.leaked); + + if r.leaked > 0 { + println!( + " highest is {}, of {} pages: {:?}", + r.highest_leaked.map_or("none".to_string(), |p| p.to_string()), + r.page_count, + r.leaked_sample + ); + println!(" `big compact {file}` rewrites the file without them"); + } + if r.file_pages != r.page_count { + println!( + " {} pages past what the meta page records ({} on disk against {})", + r.beyond_meta, r.file_pages, r.page_count + ); + println!(" a crash between a meta flip and a truncation leaves these; harmless, and `compact` clears them"); + } + + // Only these two are corruption rather than waste, and only these two fail the command. + if r.double_allocated > 0 || r.dangling > 0 { + return Err(format!( + "{file} is corrupt: {} page(s) handed out twice ({:?}), {} reference(s) past the end of the file", + r.double_allocated, r.double_allocated_sample, r.dangling + )); + } + Ok(()) +} + /// Opening a path that does not exist would *create* an empty database - the pager opens with /// `create(true)` because that is what a server wants on first start. A tool that inspects or /// copies must not, so the check is here rather than there. diff --git a/crates/big-bin/src/serve.rs b/crates/big-bin/src/serve.rs index 42531e0..485d0c0 100644 --- a/crates/big-bin/src/serve.rs +++ b/crates/big-bin/src/serve.rs @@ -31,6 +31,7 @@ use big_embed::Api; use big_http::{log, Auth, Server, ServerConfig}; use big_tls::TlsConfig; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; const DEFAULT_ADDR: &str = "127.0.0.1:7654"; @@ -67,6 +68,28 @@ usage: big serve [addr] [options] --queue connections allowed to wait; past this, 503 --read-timeout how long a client may take to send a request --query-timeout wall-clock budget for one query; 0 means none + --reclaim give trailing free pages back to the filesystem while serving. + Copy-on-write leaves holes and the freelist reuses them, so a + file that has churned is mostly free space it never hands back; + without this only an offline `big compact` shrinks it. Acts once + a quarter of the file is reclaimable, needs a moment with no + query in flight, and takes the write lock for one truncation. + Watch big_pages_reclaimed_total and big_reclaim_blocked_total + --elect-schema-leader let the agreement give the row-key namespace to another node + when the one holding it has been silent for fifteen seconds. + Without it, `bigctl cluster schema-leader` is the only way it + moves, and a dead leader means no *new* row key can be assigned + until it is back. With it, a failover copies every row key of + every table to the successor - and can burn row ids the dead + leader assigned to writes that never landed, so the deposed node + must be repaired before it is trusted again. See + docs/clustering.md. Needs --cluster + --balance let the cluster reshape itself: a node that is draining has its + ranges moved away, a node with nothing is given the tail, and a + node much fuller than another gives a range up. One step at a + time, decided by the agreement's leader. Off unless passed - a + cluster that reshapes itself unasked is one whose shape an + operator cannot predict. Needs --cluster to mean anything --max-row-keys refuse to invent more than n row keys; 0 means no limit every row key is resident in memory in both directions, so this is the ceiling on the one allocation that grows with @@ -157,7 +180,43 @@ pub fn main(args: &[String]) -> std::io::Result<()> { refuse_an_open_port(&server, bound, &opts); announce(&server, bound, &opts); - server.serve() + // **Stopped by a signal, not killed by one.** `SIGTERM` is what every supervisor sends + // first - `docker stop`, a pod being evicted, `systemctl stop` - and the default disposition + // ends the process mid-request: the workers die with sockets open, and the agreement learns + // this node has gone only when its heartbeats stop. Standing down instead finishes what is + // in flight, takes nothing more, and leaves the agreement before the port closes. + // + // Two things this does *not* promise, said here rather than found out. The agreement is + // stopped before the workers drain, so a request that arrives during the drain for a range + // this node was serving gets `503 not_serving` - correct, because a node standing down must + // not answer for a range it may already have lost, but not the same as "every request + // completes". And a move this node was driving as the agreement's leader is not finished + // for it: the range stays marked moving until an operator cancels or another leader + // finishes it. A signal narrows that window; it does not close it. + server.serve_while(stand_down_on_signal()) +} + +/// Installs the handlers that ask the server to stand down, and hands back the flag they clear. +/// +/// **One of the two `unsafe` blocks in this crate** - the other is `tty::Echo` - and the whole +/// of it is two `libc::signal` calls. The handler does exactly one thing: a relaxed store into a +/// static `AtomicBool`, which is on the short list of what a signal handler may do. Nothing is +/// allocated, locked, or printed from it; the accept loop notices the flag on its next poll. +#[allow(unsafe_code)] +fn stand_down_on_signal() -> &'static AtomicBool { + static RUNNING: AtomicBool = AtomicBool::new(true); + extern "C" fn stand_down(_: libc::c_int) { + RUNNING.store(false, Ordering::Relaxed); + } + // SAFETY: `stand_down` is an `extern "C"` function with the signature `signal` expects, it + // touches only a `static AtomicBool` through an atomic store (async-signal-safe), and the + // static lives for the whole program, so the handler can never run against freed memory. + // `signal` itself has no memory-safety preconditions beyond a valid handler pointer. + unsafe { + libc::signal(libc::SIGTERM, stand_down as *const () as libc::sighandler_t); + libc::signal(libc::SIGINT, stand_down as *const () as libc::sighandler_t); + } + &RUNNING } /// The certificate this listener presents, or the decision to serve in the clear. @@ -289,7 +348,20 @@ fn assemble( // two. let state = format!("{}.raft", opts.path); eprintln!("big: agreement state in {state}"); - Cluster::new(api, config, tls, Box::new(big_cluster::raft::FileStore::new(state))) + let leases = big_cluster::controller::Leases { + move_schema_after: opts + .elect_schema_leader + .then_some(big_cluster::controller::Leases::SCHEMA_FAILOVER), + ..big_cluster::controller::Leases::default() + }; + Cluster::with_timing( + api, + config, + tls, + Box::new(big_cluster::raft::FileStore::new(state)), + big_cluster::raft::Timing::default(), + leases, + ) } /// The listener's settings: the defaults, with whatever the operator overrode. @@ -303,6 +375,11 @@ fn serving(auth: Auth, tls: Option, opts: &Options) -> ServerConfig { workers: opts.workers.unwrap_or(base.workers), queue_depth: opts.queue.unwrap_or(base.queue_depth), read_timeout: opts.read_timeout.unwrap_or(base.read_timeout), + // Only the switch. The thresholds stay at what the policy ships with until somebody + // has a reason to move them; three knobs would triple what has to be tested for a + // setting nobody has yet needed to change. + balance: big_cluster::balance::Policy { enabled: opts.balance, ..base.balance }, + reclaim: opts.reclaim, ..base } } @@ -376,6 +453,39 @@ fn announce(server: &Server, bound: std::net::SocketAddr, Some(d) => eprintln!("big: POST /admin/backup writes into {d}"), None => eprintln!("big: no --backup-dir; POST /admin/backup is not configured"), } + // Printed on every start, both ways: whether a cluster was allowed to reshape itself is + // the first thing an operator asks after a range has moved, and a line that only appears + // when the flag was passed is one they have to remember the absence of. + match (opts.balance, opts.cluster.is_some()) { + (true, true) => eprintln!("big: balancer on; the agreement's leader reshapes the cluster"), + // Not refused: the flag is harmless alone, and a deployment that sets it everywhere + // and adds --cluster later should not have to change twice. But a switch that does + // nothing has to say so, or somebody will wait for it. + (true, false) => eprintln!("big: --balance without --cluster: nothing to balance"), + (false, _) => eprintln!("big: balancer off; POST /admin/cluster/rebalance is one step"), + } + // Both ways again, and this one is about the file rather than the cluster: an operator + // asking why a database is not shrinking should be able to read the answer out of a log. + if opts.reclaim { + eprintln!("big: reclaiming trailing free pages while serving, once a quarter is free"); + } else { + eprintln!("big: no --reclaim; the file gives space back only to an offline `big compact`"); + } + // Printed both ways for the reason the balancer is, and with the consequence spelled out: + // an automatic handover can burn row ids the dead leader promised to writes that never + // landed, which is not something to find out from a doc after the fact. + match (opts.elect_schema_leader, opts.cluster.is_some()) { + (true, true) => eprintln!( + "big: the agreement may move the row-key namespace after 15s of silence; a \ + deposed leader must be repaired before it is trusted again" + ), + (true, false) => { + eprintln!("big: --elect-schema-leader without --cluster: nothing to elect") + } + (false, _) => eprintln!( + "big: row-key namespace stays put; bigctl cluster schema-leader is the only mover" + ), + } if !server.config().auth.is_enabled() { eprintln!("big: no authentication; this port must not be reachable from anywhere else"); } @@ -405,6 +515,12 @@ struct Options { queue: Option, read_timeout: Option, query_timeout: Option, + /// Whether the agreement's leader may reshape the cluster on its own. + balance: bool, + /// Whether the agreement may give the row-key namespace to another node on its own. + elect_schema_leader: bool, + /// Whether this node may hand trailing free pages back to the filesystem while serving. + reclaim: bool, durability: Option, cluster: Option, node: Option, @@ -432,6 +548,9 @@ impl Default for Options { queue: None, read_timeout: None, query_timeout: None, + balance: false, + elect_schema_leader: false, + reclaim: false, durability: None, cluster: None, node: None, @@ -512,6 +631,18 @@ impl Options { out.query_timeout = (secs > 0).then(|| Duration::from_secs(secs)); i += 2; } + "--balance" => { + out.balance = true; + i += 1; + } + "--elect-schema-leader" => { + out.elect_schema_leader = true; + i += 1; + } + "--reclaim" => { + out.reclaim = true; + i += 1; + } "--durability" => { let v = value()?; out.durability = Some(big_db::Durability::parse(&v).ok_or_else(|| { diff --git a/crates/big-bin/src/tty.rs b/crates/big-bin/src/tty.rs index cb3b072..89d3308 100644 --- a/crates/big-bin/src/tty.rs +++ b/crates/big-bin/src/tty.rs @@ -86,9 +86,10 @@ struct Echo { } impl Echo { - /// **The only `unsafe` in this crate**, and the whole of it is four `libc` calls around a - /// `termios` struct. The crate is `#![deny(unsafe_code)]`, so this says so out loud rather - /// than the deny being quietly relaxed at the top of the file where nobody would see it. + /// **One of the two `unsafe` blocks in this crate** - the other installs the stop signal in + /// `serve.rs` - and the whole of it is four `libc` calls around a `termios` struct. The + /// crate is `#![deny(unsafe_code)]`, so this says so out loud rather than the deny being + /// quietly relaxed at the top of the file where nobody would see it. #[allow(unsafe_code)] #[cfg(unix)] fn off() -> io::Result { diff --git a/crates/big-bin/tests/serve.rs b/crates/big-bin/tests/serve.rs new file mode 100644 index 0000000..d998fcc --- /dev/null +++ b/crates/big-bin/tests/serve.rs @@ -0,0 +1,112 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The daemon as a process: the built `big` binary, a real port, and a real signal. +//! +//! Everything else in this crate's tests drives the server in-process, which is right for the +//! client and wrong for the two things only a process has - a command line and a signal +//! disposition. What is asserted here could not be asserted any other way: that `SIGTERM` +//! ends the daemon with an exit code rather than with a signal, and that a flag typed on the +//! command line reaches the line the daemon prints about itself. + +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +fn free_port() -> SocketAddr { + TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap() +} + +/// One request, one status line. Enough to know whether anybody is listening. +fn status_of(addr: SocketAddr, target: &str) -> Option { + let mut s = TcpStream::connect_timeout(&addr, Duration::from_millis(200)).ok()?; + s.set_read_timeout(Some(Duration::from_secs(2))).ok()?; + write!(s, "GET {target} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").ok()?; + let mut reply = String::new(); + s.read_to_string(&mut reply).ok()?; + reply.split_whitespace().nth(1)?.parse().ok() +} + +fn waiting(what: &str, budget: Duration, mut check: impl FnMut() -> bool) { + let started = Instant::now(); + while started.elapsed() < budget { + if check() { + return; + } + std::thread::sleep(Duration::from_millis(25)); + } + panic!("gave up waiting for {what} after {:?}", started.elapsed()); +} + +/// **`SIGTERM` is a request to stand down, and it is honoured.** A supervisor sends it first - +/// `docker stop`, a pod being evicted, `systemctl stop` - and a daemon that dies of it rather +/// than answering it leaves sockets open and an agreement that only finds out from silence. +/// The tell is the exit status: a process killed by a signal has none, and one that stood down +/// has zero. +#[test] +fn sigterm_stands_the_daemon_down_rather_than_killing_it() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("data.big"); + let log = std::fs::File::create(dir.path().join("stderr")).unwrap(); + let addr = free_port(); + + let mut daemon = Command::new(env!("CARGO_BIN_EXE_big")) + .arg("serve") + .arg(&db) + .arg(addr.to_string()) + // On the command line so that the line it produces can be read back below. + .arg("--balance") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::from(log)) + .spawn() + .expect("the built daemon starts"); + + waiting("the daemon to answer", Duration::from_secs(15), || { + status_of(addr, "/health") == Some(200) + }); + + let told = Command::new("kill") + .args(["-TERM", &daemon.id().to_string()]) + .status() + .expect("`kill` is on the path"); + assert!(told.success(), "the signal was sent"); + + let status = { + let started = Instant::now(); + loop { + if let Some(s) = daemon.try_wait().unwrap() { + break s; + } + assert!( + started.elapsed() < Duration::from_secs(15), + "the daemon did not stand down within fifteen seconds of SIGTERM" + ); + std::thread::sleep(Duration::from_millis(25)); + } + }; + assert_eq!( + status.code(), + Some(0), + "stood down with an exit code, not killed by the signal: {status}" + ); + assert_eq!(status_of(addr, "/health"), None, "and the port is closed"); + + // The flag reached the announcement. Alone it does nothing, and it says so rather than + // letting an operator wait for a balancer with nothing to balance. + let said = std::fs::read_to_string(dir.path().join("stderr")).unwrap(); + assert!(said.contains("big serving"), "{said}"); + assert!(said.contains("--balance without --cluster"), "{said}"); +} diff --git a/crates/big-cluster/src/admin.rs b/crates/big-cluster/src/admin.rs index e0fad56..45b214a 100644 --- a/crates/big-cluster/src/admin.rs +++ b/crates/big-cluster/src/admin.rs @@ -275,10 +275,15 @@ impl Cluster

{ fn abandon(&self, id: RangeId, why: ClusterError) -> ClusterError { match self.with_map(|m| m.cancel_move(id).map_err(|e| e.to_string())) { Ok(_) => why, - Err(e) => ClusterError::Refused(format!( - "{why} - and the move could not be called off either ({e}), so range {id} is \ - still marked as moving. `cluster cancel {id}` clears it" - )), + Err(e) => { + // Counted as well as said: an operator reads the sentence, a background loop + // drops it, and the range it names silences every balancing step from here on. + self.counters.move_cancel_failed(); + ClusterError::Refused(format!( + "{why} - and the move could not be called off either ({e}), so range {id} \ + is still marked as moving. `cluster cancel {id}` clears it" + )) + } } } @@ -345,8 +350,10 @@ impl Cluster

{ /// /// **A learner, not a voter.** A node that has just arrived holds no range and has not /// caught up on the log; counting it towards a majority would raise the bar for every - /// election while it contributed nothing to one. The balancer promotes it once it has - /// something to serve - or an operator does, with `admit`. + /// election while it contributed nothing to one. With the balancer on, the agreement's + /// leader admits it once it answers for data and holds the log to within the compaction + /// margin; otherwise an operator does, with `admit`. Either way it is given nothing to + /// serve until it counts. pub fn add_node(&self, name: &str, addr: &str) -> Result<()> { let mut next = self.members(); if let Some(existing) = next.iter_mut().find(|m| m.name == name) { @@ -467,13 +474,16 @@ impl Cluster

{ /// 1. **The keys go first**, every table's, and they are not scoped to a range - a row key /// has to mean the same number in every shard, so the successor needs the whole mapping /// or it will invent a second id for a string that already has one. - /// 2. **The floor goes with them.** A leader hands out record ids from a number held in - /// memory and not yet written anywhere; a successor that started from what is *on disk* - /// would hand out ids the old leader has already given away. This is the part that has - /// no second chance - the ids are gone by the time anybody notices. - /// 3. **Then the decision commits**, and only then does the successor answer. Until it - /// does, the old leader is still the one interning, which is what keeps the window from - /// being one in which nobody is. + /// 2. **The old leader stops.** It is told the epoch the move read and refuses to intern or + /// allocate until the map is past it. The window that opens here is one in which nobody + /// interns, and a write with a new key waits; the window it closes is one in which two + /// nodes do, which no write could ever wait out. + /// 3. **The floor goes next.** A leader hands out record ids from a number held in memory + /// and not yet written anywhere; a successor that started from what is *on disk* would + /// hand out ids the old leader has already given away. This is the part that has no + /// second chance - the ids are gone by the time anybody notices - and it is read only + /// once nothing can still be raising it. + /// 4. **Then the decision commits**, and only then does the successor answer. pub fn move_schema_leader(&self, to: &str) -> Result<()> { let target = self.node_named(to)?; let source = self.schema_leader(); @@ -489,20 +499,105 @@ impl Cluster

{ self.push_keys(target, &table.name, keys)?; } - // 2. The floor: one past the highest id the old leader has handed out, whether or not - // it has landed. Taken *after* the keys, so nothing allocated during the copy is - // missed. + // 2. The old leader stops. From here until the decision lands nobody interns, which + // is the gap this move is allowed to have; the one it is not allowed to have is two + // nodes interning at once, and a source still answering while its floor is read - or + // after, to a coordinator holding the map from before the move - is exactly that. A + // source that cannot be told stops the move: refused, not raced. + let epoch = self.map().epoch; + self.step_down(source, epoch)?; + + // 3. The floor: one past the highest id the old leader has handed out, whether or not + // it has landed. Taken *after* the keys and *after* the source stopped, so it is a + // number nothing is still adding to. let floors = self.allocation_floors(source)?; self.seed_floors(target, &floors)?; - // 3. The decision. + // 4. The decision. Ready at once: the keys and the floor went across above, by hand, + // which is the whole of what a successor has to take over. self.with_map(|m| { m.schema_leader = target; + m.schema_ready = true; Ok(()) }) .map(|_| ()) } + /// Finishes a handover the agreement decided: gives the successor every row key the + /// survivors hold, then marks it ready. + /// + /// **Run by the agreement's leader, from every node it can reach.** The old leader is not + /// among them - that is why there is a handover - so the mapping is rebuilt from the + /// copies every owner took when it was told what a key meant. A key the old leader + /// interned for a write that never landed anywhere is the one thing this cannot recover; + /// the successor will hand that row id to a different string, and the deposed node, if + /// it ever comes back, will refuse the contradiction when it is repaired. That is the + /// residual risk of an automatic failover, and it is stated in `docs/clustering.md` + /// rather than hidden. + /// + /// **A contradiction among the survivors stops it.** Two live nodes disagreeing about + /// what a row id means is a cluster that has already diverged; marking the successor + /// ready would let it pick one side and write on it. Nothing interns until somebody + /// looks, and `big_cluster_schema_handover_blocked` is what makes them. + pub fn finish_schema_handover(&self) -> Result> { + let map = self.map(); + if map.schema_ready { + return Ok(None); + } + let successor = map.schema_leader; + let this = self.config.this_index(); + let outcome = self.hand_over_to(successor, this); + self.handover_blocked.store( + matches!(&outcome, Err(e) if !e.is_unreachable()), + std::sync::atomic::Ordering::Relaxed, + ); + outcome?; + self.with_map(|m| { + if m.schema_leader != successor { + return Err("the namespace moved again while it was being handed over".into()); + } + m.schema_ready = true; + Ok(()) + })?; + Ok(Some(self.name_of_agreed(successor))) + } + + /// Every row key every reachable member holds, pushed to `successor`. + fn hand_over_to(&self, successor: usize, this: usize) -> Result<()> { + // The successor's schema has to exist first, or a key naming a field it has never + // heard of has nowhere to land. Every node carries the same schema - DDL fans out - + // so this node's is as good as any. + self.match_schema(this, successor)?; + let tables = self.pull_schema(this)?.0; + for (i, member) in self.members().iter().enumerate() { + if !member.reachable() || i == successor { + continue; + } + for table in &tables { + let keys = match self.pull_keys(i, &table.name) { + Ok(keys) => keys, + // A survivor that is not answering right now contributes nothing, and + // that is not a contradiction: the next pass will ask it again. + Err(e) if e.is_unreachable() => continue, + Err(e) => return Err(e), + }; + if !keys.is_empty() { + self.push_keys(successor, &table.name, keys)?; + } + } + } + Ok(()) + } + + /// Tells a node to stop leading the schema until the map has moved past `epoch`. + fn step_down(&self, node: usize, epoch: u64) -> Result<()> { + if node == self.config.this_index() { + self.stand_down_schema(epoch); + return Ok(()); + } + self.ask(node, path::SCHEMA_STEP_DOWN, &wire::put_u64_body(epoch), None).map(|_| ()) + } + /// One past the highest record id a node has handed out for each table, landed or not. fn allocation_floors(&self, node: usize) -> Result> { if node == self.config.this_index() { @@ -515,18 +610,74 @@ impl Cluster

{ /// This node's own floors, which only the schema leader has anything in. pub fn floors_here(&self) -> Vec<(String, RecordId)> { let allocated = self.allocated.lock().unwrap_or_else(|e| e.into_inner()); + let map = self.map(); let mut out: Vec<(String, RecordId)> = Vec::new(); for table in self.schema() { - // The greater of what is on disk and what has been promised. A leader that has - // never allocated still has a floor - it is just the data's own high-water mark. + // The greatest of what is on disk, what has been promised, and what the agreement + // has been told may be promised. A leader that has never allocated still has a + // floor - it is just the data's own high-water mark. let landed = self.api.max_record(&table.name).ok().flatten().map_or(0, |m| m.saturating_add(1)); let promised = allocated.get(&table.name).copied().unwrap_or(0); - out.push((table.name.clone(), landed.max(promised))); + let reserved = map.reserved_for(&table.name); + out.push((table.name.clone(), landed.max(promised).max(reserved))); } out } + /// Raises the agreement's ceiling on the record ids `table` may be handed, to `upto`. + /// + /// **Through the agreement's leader, whoever that is.** The schema leader and the + /// agreement's leader are two roles that usually sit on two nodes, and only the second can + /// propose - so this is one hop when they differ, amortised over a block of ids. Waits + /// until this node has applied the result: a ceiling that has been proposed and not landed + /// is one a successor could still start below. + /// + /// A cluster with no agreement has nothing to commit to and nothing to be succeeded by; + /// its floor in memory is all there is, and all there needs to be. + pub(super) fn reserve_ids(&self, table: &str, upto: RecordId) -> Result<()> { + let Some(controller) = &self.controller else { return Ok(()) }; + if controller.is_leader() { + return self.reserve_here(table, upto).map(|_| ()); + } + let Some(leader) = controller.leader() else { + return Err(ClusterError::Refused( + "no node leads the agreement right now, so no record ids can be reserved; \ + try again once an election has finished" + .to_string(), + )); + }; + let body = wire::put_floors(&[(table.to_string(), upto)]); + self.ask(leader, path::RESERVE, &body, None)?; + self.await_reserved(table, upto) + } + + /// The agreement leader's half of [`Cluster::reserve_ids`]: commits the ceiling. + /// + /// Public because the peer route calls it. Answers the epoch it landed at. + pub fn reserve_here(&self, table: &str, upto: RecordId) -> Result { + self.with_map(|m| { + m.reserve(table, upto); + Ok(()) + }) + } + + /// Waits until this node's own map carries a ceiling of at least `upto` for `table`. + fn await_reserved(&self, table: &str, upto: RecordId) -> Result<()> { + let deadline = Instant::now() + PROPOSAL_TIMEOUT; + while Instant::now() < deadline { + if self.map().reserved_for(table) >= upto { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(5)); + } + Err(ClusterError::Refused(format!( + "the agreement's leader took the reservation but it has not reached this node \ + after {}s; nothing was handed out", + PROPOSAL_TIMEOUT.as_secs() + ))) + } + /// Starts a node's floors at least as high as these. fn seed_floors(&self, node: usize, floors: &[(String, RecordId)]) -> Result<()> { if node == self.config.this_index() { @@ -567,7 +718,8 @@ impl Cluster

{ .filter_map(|t| self.api.max_record(&t.name).ok().flatten()) .max() .map_or(0, |m| m.saturating_add(1)); - crate::balance::NodeLoad { pages: Some(pages), frontier } + // This node holds its own log, whatever else is true of it. + crate::balance::NodeLoad { pages: Some(pages), frontier, caught_up: true } } /// What every node weighs, in member order. @@ -584,13 +736,21 @@ impl Cluster

{ if i == self.config.this_index() { return self.load(); } - match self.ask(i, path::LOAD, &[], None) { - Err(_) => crate::balance::NodeLoad::default(), - Ok(bytes) => match wire::get_load(&bytes) { - Err(_) => crate::balance::NodeLoad::default(), - Ok((pages, frontier)) => { - crate::balance::NodeLoad { pages: Some(pages), frontier } - } + // **A node that does not answer is `None`, not zero** - and it is counted, + // because to the balancer it is a node that can neither give nor take, and a + // transient timeout and a dead machine would otherwise be the same silence. + match self + .ask(i, path::LOAD, &[], None) + .and_then(|b| self.read(i, || wire::get_load(&b))) + { + Err(_) => { + self.counters.load_unanswered(); + crate::balance::NodeLoad::default() + } + Ok((pages, frontier)) => crate::balance::NodeLoad { + pages: Some(pages), + frontier, + caught_up: self.controller.as_ref().is_some_and(|c| c.caught_up(i)), }, } }) @@ -603,21 +763,37 @@ impl Cluster

{ /// move is a moment where a query can fail*. So this never chains two changes, and a /// cluster that needs three moves takes three calls - each one against facts gathered /// afresh, rather than against a plan made before the first move happened. - pub fn rebalance(&self, policy: &crate::balance::Policy) -> Result> { + pub fn rebalance(&self, policy: &crate::balance::Policy) -> Result> { let (map, members, loads) = (self.map(), self.members(), self.loads()); let Some(action) = crate::balance::plan(&map, &members, &loads, policy) else { return Ok(None); }; - match action { + let done = match action { + crate::balance::Action::Admit { node } => { + self.admit(&node).map(|_| Balanced::Admitted { node }) + } crate::balance::Action::SplitTail { at, to } => { - self.split_range(at, Some(&to))?; - Ok(Some(format!("split the tail at {at} and gave it to `{to}`"))) + self.split_range(at, Some(&to)).map(|_| Balanced::SplitTail { at, to }) } crate::balance::Action::Move { range, to } => { - let report = self.move_range(range, &to)?; - Ok(Some(format!("moved shards {} to `{to}`", report.shards))) + self.move_range(range, &to).map(Balanced::Moved) + } + }; + // Counted here rather than by the caller, so the operator's one step and the steward's + // many are one number - and so that a move whose source could not let go is a number + // at all, which as a field on a report handed to a loop it was not. + match &done { + Ok(Balanced::Admitted { .. }) => self.counters.balance_admit(), + Ok(Balanced::SplitTail { .. }) => self.counters.balance_split(), + Ok(Balanced::Moved(report)) => { + self.counters.balance_move(); + if !report.dropped { + self.counters.balance_drop_failed(); + } } + Err(_) => self.counters.balance_error(), } + done.map(Some) } /// What the cluster looks like right now: every range, who holds it, and what is moving. @@ -629,7 +805,7 @@ impl Cluster

{ Topology { epoch: map.epoch, leader: self.controller.as_ref().and_then(|c| c.leader()).and_then(|n| self.name_of(n)), - schema_leader: self.name_of(map.schema_leader).unwrap_or_default(), + schema_leader: self.name_of_agreed(map.schema_leader), members: self .members() .iter() @@ -680,6 +856,33 @@ pub struct MoveReport { pub outcome: String, } +/// What one balancing step did. +/// +/// A value rather than a sentence, so that the caller who wants the sentence gets the same one +/// it always got and the caller who wants to count gets something to count. The move carries +/// its whole report: `dropped` in particular is a fact a background loop must not lose. +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum Balanced { + /// A learner that had caught up was made a full member. + Admitted { node: String }, + /// The tail was cut above everything written and the empty half given to a node with + /// nothing. No bytes moved. + SplitTail { at: big_engine::ShardId, to: String }, + /// A populated range was handed to another node. + Moved(MoveReport), +} + +impl Balanced { + /// The sentence `POST /admin/cluster/rebalance` has always answered with. + pub fn describe(&self) -> String { + match self { + Self::Admitted { node } => format!("admitted `{node}` as a full member"), + Self::SplitTail { at, to } => format!("split the tail at {at} and gave it to `{to}`"), + Self::Moved(r) => format!("moved shards {} to `{}`", r.shards, r.to), + } + } +} + /// The cluster's shape, as a report. #[derive(Clone, PartialEq, Eq, Debug)] pub struct Topology { diff --git a/crates/big-cluster/src/balance.rs b/crates/big-cluster/src/balance.rs index 46818f0..fbfe9e1 100644 --- a/crates/big-cluster/src/balance.rs +++ b/crates/big-cluster/src/balance.rs @@ -43,6 +43,13 @@ pub struct NodeLoad { /// One past the highest record id the node holds, over every table. What a tail split is /// cut above. pub frontier: u64, + /// Whether the node holds the agreement's log to within a margin of its end. + /// + /// What decides whether a learner may become a member. A node still replaying what it + /// missed would raise the bar for every election while being able to help decide none - + /// the exact reason a learner is not a voter to begin with. Only the leader knows this, + /// which is fine: only the leader proposes. + pub caught_up: bool, } /// When the balancer is allowed to act, and how hard. @@ -68,6 +75,12 @@ impl Default for Policy { /// The one thing to do next. #[derive(Clone, PartialEq, Eq, Debug)] pub enum Action { + /// Make a learner that has caught up a full member, so that it votes and may hold a range. + /// + /// **A change to who the cluster is, not to who holds what** - and the only one the + /// balancer makes. It has to come first: until the node is a member, nothing below can see + /// it as a node with nothing to give it the tail. + Admit { node: String }, /// Cut the tail above everything written and give the empty half to a node with nothing. /// /// **The scale-out that moves no bytes**, and the reason it is preferred: it costs one @@ -81,10 +94,12 @@ pub enum Action { /// /// The order is the policy, and the first that applies wins: /// -/// 1. **A node that is draining** gives up a range. An operator asked for this, so it outranks +/// 1. **A learner that is answering and has caught up** is admitted. It joined to take work, +/// and nothing below can give it any until it counts. +/// 2. **A node that is draining** gives up a range. An operator asked for this, so it outranks /// anything the balancer noticed by itself. -/// 2. **A node with nothing** is given the tail, cut above everything written. No bytes move. -/// 3. **A spread that is too wide** moves one range from the fullest node to the emptiest. +/// 3. **A node with nothing** is given the tail, cut above everything written. No bytes move. +/// 4. **A spread that is too wide** moves one range from the fullest node to the emptiest. /// /// `None` whenever anything is already in flight: a second decision made about a map that has /// not settled is a decision about a state that is already gone. @@ -104,6 +119,18 @@ pub fn plan( } let name = |n: usize| members.get(n).map(|m: &Member| m.name.clone()); + + // 1. Somebody joined, and is ready to count. Answering `/internal/load` is what proves it + // is up for data and not only for the log; caught up is what proves counting it will not + // stall the next election. + for (i, m) in members.iter().enumerate() { + if !matches!(m.state, crate::raft::MemberState::Learner) { + continue; + } + if load.get(i).is_some_and(|l| l.pages.is_some() && l.caught_up) { + return Some(Action::Admit { node: name(i)? }); + } + } // A node that did not answer is not a candidate in either direction: it may be full, it may // be empty, and sending work to a machine that is not there is the worse guess. let live = |n: usize| { @@ -112,7 +139,7 @@ pub fn plan( }; let pages = |n: usize| load.get(n).and_then(|l| l.pages).unwrap_or(0); - // 1. Somebody asked for this. + // 2. Somebody asked for this. for (i, m) in members.iter().enumerate() { if !matches!(m.state, crate::raft::MemberState::Draining) { continue; @@ -124,7 +151,7 @@ pub fn plan( return Some(Action::Move { range: map.ranges[range].id, to: name(to)? }); } - // 2. A node that holds nothing. Give it the part of the space nothing has been written to + // 3. A node that holds nothing. Give it the part of the space nothing has been written to // yet, which is the only thing that can change hands without a copy. if let Some(empty) = (0..members.len()).find(|n| live(*n) && map.held_by(*n).is_empty()) { // Above every record any node holds, so the half handed over is empty by construction. @@ -138,7 +165,7 @@ pub fn plan( } } - // 3. A spread that has grown too wide. + // 4. A spread that has grown too wide. let fullest = (0..members.len()).filter(|n| live(*n)).max_by_key(|n| pages(*n))?; let emptiest = (0..members.len()).filter(|n| live(*n)).min_by_key(|n| pages(*n))?; if fullest == emptiest { diff --git a/crates/big-cluster/src/config.rs b/crates/big-cluster/src/config.rs index 5c8aec6..82468dd 100644 --- a/crates/big-cluster/src/config.rs +++ b/crates/big-cluster/src/config.rs @@ -661,7 +661,14 @@ impl ClusterConfig { text(&node.shards.to_string(), &mut h); eat(&node.replica_of.map_or(u64::MAX, |r| r as u64).to_le_bytes(), &mut h); } - text(&self.nodes[self.leader].name.clone(), &mut h); + // **The schema leader is deliberately not folded in.** It used to be, and it was right + // while the leader was a name read once from a file: two files naming different ones + // described two different clusters. It is a field in the map now, moved by an operator + // and - once a lease allows it - by the agreement itself. Two nodes whose files were + // updated at different moments after a *correct* failover would stop being able to + // talk to each other, which is a successful failover causing a partition: the worst + // failure available. The map overrides the file within a heartbeat anyway, so folding + // it in protected nothing. h } @@ -746,7 +753,15 @@ impl ClusterConfig { .collect(); // Nothing is behind before anything has happened, which is exactly what the file // asserts by naming a primary for every range. - crate::raft::RangeMap { epoch: 0, ranges, stale: Vec::new(), schema_leader: self.leader } + crate::raft::RangeMap { + epoch: 0, + ranges, + stale: Vec::new(), + schema_leader: self.leader, + reserved: Vec::new(), + // The file's leader has nothing to take over: it starts holding the namespace. + schema_ready: true, + } } /// The members the cluster starts from, in the file's order. diff --git a/crates/big-cluster/src/controller.rs b/crates/big-cluster/src/controller.rs index 8adb461..eddb6c4 100644 --- a/crates/big-cluster/src/controller.rs +++ b/crates/big-cluster/src/controller.rs @@ -56,6 +56,16 @@ pub struct Leases { /// stopped before the new one starts, and the only thing making that true is this /// inequality plus clocks that run at similar rates. pub promote_after: Duration, + /// The leader gives the row-key namespace to another node after this long without hearing + /// from the one that holds it. `None` leaves that to an operator. + /// + /// **Much longer than `promote_after`, and off by default.** Moving the namespace copies + /// every row key of every table to the successor, which is the most expensive thing the + /// agreement can decide to do - so it waits until ranges have already failed over and the + /// map has settled, and it does not happen at all unless somebody turned it on. The same + /// lease the schema leader serves under (`serve_for`) is what makes this safe: by the time + /// the agreement moves the namespace, the old holder has long since stopped acting on it. + pub move_schema_after: Option, } impl Default for Leases { @@ -63,10 +73,25 @@ impl Default for Leases { Self { serve_for: Duration::from_millis(1_500), promote_after: Duration::from_millis(4_500), + move_schema_after: None, } } } +impl Leases { + /// How long the agreement waits before moving the namespace, when it is allowed to. + /// + /// Three and a third times `promote_after`: strictly longer is the property that matters, + /// and this much longer is the margin for a range failover to have landed first. + pub const SCHEMA_FAILOVER: Duration = Duration::from_secs(15); +} + +/// The lease of a node that has not yet heard from the agreement at all. +/// +/// A sentinel rather than a zero because the clock it is compared against also starts at zero: +/// a fresh node would otherwise read its own silence as having just been in touch. +const NEVER: u64 = u64::MAX; + /// One node's half of the agreement, and the thread that drives it. pub struct Controller { raft: Mutex, @@ -95,6 +120,11 @@ pub struct Controller { roster: RwLock>, /// Milliseconds since this process started, at the last moment this node could prove it /// was still in touch with a majority. The lease, in one number. + /// + /// [`NEVER`] until it has proved that once. **Not zero**: this clock starts at zero too, so + /// zero reads as "heard from the agreement just now" on a node that has heard from nobody + /// at all - which is exactly a node that has restarted and may already have been replaced + /// while it was away. lease_at: AtomicU64, started: Instant, inbox: mpsc::Sender, @@ -211,14 +241,17 @@ impl Controller { // than taken from the file alone, or a node that restarted would come back believing // the file's map - which, once a range has moved, describes a cluster that no longer // exists. Members follow every entry because a configuration change applies when it is - // appended; ranges follow only committed ones. + // appended; ranges follow only committed ones - and "committed" is a log position, so + // the vector's position is offset by the base, or a compacted log would apply a map no + // majority had agreed to yet. let mut members = config.seed_members(); { let mut map = ranges.write().expect("no panic holds this lock"); for (i, entry) in raft.log().iter().enumerate() { + let index = raft.base() + i as u64; match &entry.decision { Decision::Members(ms) => members = ms.clone(), - Decision::Ranges(m) if i as u64 <= raft.commit_index() => *map = m.clone(), + Decision::Ranges(m) if index <= raft.commit_index() => *map = m.clone(), _ => {} } } @@ -243,7 +276,7 @@ impl Controller { this, peers: Arc::clone(&peers), roster: RwLock::new(None), - lease_at: AtomicU64::new(0), + lease_at: AtomicU64::new(NEVER), started, inbox: tx, outbox, @@ -281,8 +314,19 @@ impl Controller { /// the new member list stops waiting while the entry is still in flight, and the next /// proposal is refused as busy. Nothing is settled until the log is. pub fn settled(&self) -> bool { + self.raft.lock().expect("no panic holds this lock").settled() + } + + /// Whether `node` holds the log to within the margin a leader keeps before compacting. + /// + /// **What lets a learner become a member.** A node further behind than that would be sent + /// a snapshot rather than entries, and counting it towards a majority while it replays one + /// would raise the bar for an election it could not help decide. Only a leader tracks what + /// its followers hold, so on any other node this is `false` - which is right, because only + /// a leader proposes the change this answer gates. + pub fn caught_up(&self, node: NodeId) -> bool { let raft = self.raft.lock().expect("no panic holds this lock"); - raft.commit_index() == raft.last_index() + raft.matched(node).is_some_and(|m| m + Raft::KEEP_ENTRIES >= raft.last_index()) } /// The map, as last committed. @@ -337,8 +381,23 @@ impl Controller { map.ranges.iter().any(|r| r.group.contains(&self.this) && r.group.len() > 1) } + /// Whether this node may act as the schema leader the map names it as. + /// + /// **Always fenced.** A range with no copy is never fenced because nothing could take it; + /// the schema leader can always be replaced, so it acts only while it has heard from the + /// agreement recently enough to know it has not been. The same lease as a range, and the + /// same margin against `promote_after` is what makes replacing it safe. + pub fn may_lead_schema(&self) -> bool { + self.since_lease() < self.leases.serve_for.as_millis() as u64 + } + fn since_lease(&self) -> u64 { - self.now().saturating_sub(self.lease_at.load(Ordering::Relaxed)) + match self.lease_at.load(Ordering::Relaxed) { + // Never in touch is not "in touch a long time ago"; it is further than any lease + // reaches, which is what a node that has just started has to assume about itself. + NEVER => NEVER, + at => self.now().saturating_sub(at), + } } /// A message from a peer, on its way to the one thread that decides. @@ -475,10 +534,24 @@ impl Controller { /// already committed - a second proposal while the first is in flight would be a decision /// made about a state that has not settled. fn promotion(&self, raft: &Raft, now: u64) -> Option { - if raft.commit_index() != raft.log().len() as u64 - 1 { + Self::promotion_for(&self.leases, raft, &self.map(), now).map(Decision::Ranges) + } + + /// The rule itself, with nothing of this node's around it. + /// + /// Public so that it can be run against a simulated agreement - a `Raft` a test has driven + /// to a chosen state - without a thread, a socket or a store. The controller only decides + /// *when* to ask; what is asked is entirely here. + pub fn promotion_for( + leases: &Leases, + raft: &Raft, + current: &RangeMap, + now: u64, + ) -> Option { + if !raft.settled() { return None; } - let promote_after = self.leases.promote_after.as_millis() as u64; + let promote_after = leases.promote_after.as_millis() as u64; // A node nobody has heard from *yet* is not a node that has gone: at startup nothing // has been heard from anybody, and marking every peer behind before the first // heartbeat would mean a cluster that has to be repaired the moment it starts. @@ -487,7 +560,7 @@ impl Controller { || now.saturating_sub(raft.last_heard(node).unwrap_or(0)) < promote_after }; - let mut next = self.map(); + let mut next = current.clone(); let mut changed = false; // Marked behind, and never unmarked here: only a repair knows whether a copy has @@ -532,18 +605,57 @@ impl Controller { // is now serving a range it no longer holds. `GET /verify` is how that is found; the // mark is what makes somebody look. + // **The schema leader, last and slowest.** Silence long enough that the ranges above + // have already failed over and the map has settled: moving the namespace copies every + // row key of every table, and moving it on a blip would be the most expensive flap + // there is. Off unless a lease says how long, which is the operator's switch. + // + // The successor is chosen the way a range's is - in a fixed order, so two leaders + // elected in sequence choose the same node - from the voters that are answering and + // not marked behind. Not required to hold a range: after a rebalance a voter may hold + // none, and the namespace has nothing to do with ranges. What it is *not* given is + // the keys: this decides, and the handover that follows it - run by the agreement's + // leader from every survivor - is what makes the successor ready. Until then nobody + // interns, which is allowed; two nodes interning is not. + // + // The deposed node is marked behind whether or not it holds a copy of anything. It + // may hold row ids it interned for writes that never landed, which the survivors have + // never seen and the successor will hand out again - so it must be repaired, which + // will refuse the contradiction loudly, before it is trusted with anything. + if let Some(after) = leases.move_schema_after { + let after = after.as_millis() as u64; + let heard = |node: NodeId| { + node == raft.id() || now.saturating_sub(raft.last_heard(node).unwrap_or(0)) < after + }; + let deposed = next.schema_leader; + if !heard(deposed) { + let members = raft.membership(); + let successor = (0..members.len()) + .find(|&n| members[n].takes_ranges() && live(n) && !next.is_stale(n)); + if let Some(successor) = successor { + next.schema_leader = successor; + next.schema_ready = false; + if !next.is_stale(deposed) { + next.stale.push(deposed); + next.stale.sort_unstable(); + } + changed = true; + } + } + } + if !changed { return None; } // A promotion never reshapes the space, so this cannot fail - but the check is here // rather than assumed, because every proposal goes through it and a proposal that // skipped it would be the one that got it wrong. - if let Err(e) = next.check() { + if let Err(e) = next.check_with(raft.membership()) { debug_assert!(false, "a promotion produced an invalid map: {e}"); return None; } next.epoch += 1; - Some(Decision::Ranges(next)) + Some(next) } /// Proposes a new map, which every node will adopt once a majority has it. @@ -556,8 +668,10 @@ impl Controller { /// committed map that leaves a gap is a record id nobody answers for and there is nothing /// downstream that would notice. pub fn propose_map(&self, mut next: RangeMap) -> core::result::Result { - next.check().map_err(ProposeError::Invalid)?; let mut raft = self.raft.lock().expect("no panic holds this lock"); + // Against the membership as well as the space: a map naming a schema leader that has + // left, or one still catching up, would commit and then nobody would intern. + next.check_with(raft.membership()).map_err(ProposeError::Invalid)?; if !raft.is_leader() { return Err(ProposeError::NotLeader { leader: raft.leader() }); } diff --git a/crates/big-cluster/src/counters.rs b/crates/big-cluster/src/counters.rs index e8a4f38..ccaff82 100644 --- a/crates/big-cluster/src/counters.rs +++ b/crates/big-cluster/src/counters.rs @@ -32,6 +32,13 @@ pub struct Counters { sent: AtomicU64, unreachable: AtomicU64, refused: AtomicU64, + load_unanswered: AtomicU64, + balance_moves: AtomicU64, + balance_splits: AtomicU64, + balance_admits: AtomicU64, + balance_errors: AtomicU64, + balance_drops_failed: AtomicU64, + move_cancel_failed: AtomicU64, } impl Counters { @@ -43,6 +50,42 @@ impl Counters { self.sent.fetch_add(1, Ordering::Relaxed); } + /// A peer that did not say what it weighs. To the balancer that node is neither a source + /// nor a destination - so a count that climbs is a cluster that will not reshape, and + /// without this number a transient timeout and a dead machine look the same. + pub fn load_unanswered(&self) { + self.load_unanswered.fetch_add(1, Ordering::Relaxed); + } + + pub fn balance_move(&self) { + self.balance_moves.fetch_add(1, Ordering::Relaxed); + } + + pub fn balance_split(&self) { + self.balance_splits.fetch_add(1, Ordering::Relaxed); + } + + pub fn balance_admit(&self) { + self.balance_admits.fetch_add(1, Ordering::Relaxed); + } + + pub fn balance_error(&self) { + self.balance_errors.fetch_add(1, Ordering::Relaxed); + } + + /// A move that landed, whose source could not let go of its copy. Space nobody reclaims + /// until somebody looks; a background loop that only ever saw the move succeed would never + /// tell them to. + pub fn balance_drop_failed(&self) { + self.balance_drops_failed.fetch_add(1, Ordering::Relaxed); + } + + /// A move that failed and could not be called off either. Each one is a range still marked + /// moving, and a range marked moving silences the balancer cluster-wide. + pub fn move_cancel_failed(&self) { + self.move_cancel_failed.fetch_add(1, Ordering::Relaxed); + } + /// A peer that did not answer. Returns the error it was given, so a call site can count and /// fail in one expression rather than in two lines that can drift apart. pub fn unreachable(&self, e: E) -> E { @@ -63,6 +106,13 @@ impl Counters { sent: self.sent.load(Ordering::Relaxed), unreachable: self.unreachable.load(Ordering::Relaxed), refused: self.refused.load(Ordering::Relaxed), + load_unanswered: self.load_unanswered.load(Ordering::Relaxed), + balance_moves: self.balance_moves.load(Ordering::Relaxed), + balance_splits: self.balance_splits.load(Ordering::Relaxed), + balance_admits: self.balance_admits.load(Ordering::Relaxed), + balance_errors: self.balance_errors.load(Ordering::Relaxed), + balance_drops_failed: self.balance_drops_failed.load(Ordering::Relaxed), + move_cancel_failed: self.move_cancel_failed.load(Ordering::Relaxed), } } } @@ -75,6 +125,20 @@ pub struct Counts { pub unreachable: u64, /// How many were answered with a refusal. pub refused: u64, + /// Peers that did not say what they weigh when the balancer asked. + pub load_unanswered: u64, + /// Ranges the balancer moved. + pub balance_moves: u64, + /// Tails the balancer cut for a node with nothing. + pub balance_splits: u64, + /// Learners the balancer made full members. + pub balance_admits: u64, + /// Balancing steps that failed. + pub balance_errors: u64, + /// Moves that landed but left their source holding a copy it could not drop. + pub balance_drops_failed: u64, + /// Moves that failed and could not be called off - ranges still marked moving. + pub move_cancel_failed: u64, } /// Everything `/metrics` renders about the cluster, taken at one moment. @@ -91,5 +155,15 @@ pub struct Snapshot { pub leader: bool, /// Copies the agreement has marked behind. **Redundancy this cluster has lost.** pub behind: usize, + /// Ranges with a move in flight. **Nonzero for longer than a move takes is the balancer + /// silenced**: it plans nothing while anything is moving, and nothing clears the mark on + /// its own. + pub moving: usize, + /// Whether the schema leader has finished taking the namespace over. `false` is a window + /// in which nobody assigns row ids; one that stays `false` is a handover that is stuck. + pub schema_ready: bool, + /// Whether the last handover found two survivors disagreeing about a row id. **Somebody + /// has to look**: nothing clears this but a handover that succeeds. + pub handover_blocked: bool, pub counts: Counts, } diff --git a/crates/big-cluster/src/error.rs b/crates/big-cluster/src/error.rs index 1ff0d2c..56bb4d2 100644 --- a/crates/big-cluster/src/error.rs +++ b/crates/big-cluster/src/error.rs @@ -66,6 +66,22 @@ pub enum ClusterError { /// would say so, which is the one failure a client could never see - so a node that cannot /// prove it is still the primary stops being one. NotServing { node: String, shards: String }, + /// A request to intern or allocate reached a node that does not lead the schema. + /// + /// Either the sender holds an older map, or this node has stood down for a move that has + /// not landed yet. Retryable, against the leader the answer names: two nodes interning at + /// once hand one string two row ids, and this refusal is what keeps it to one. + NotSchemaLeader { node: String, leader: String, epoch: u64 }, + /// The map names this node as the schema leader and it has lost touch with the agreement. + /// + /// It cannot know it has not been replaced, so it stops - `NotServing`, for the one role + /// that is not a range. A write introducing a new key waits; everything else is untouched. + SchemaLeaseLost { node: String }, + /// The namespace has just moved and its new holder has not finished taking it over. + /// + /// Nobody interns in this window - the old leader has been deposed, the new one does not + /// yet hold every row key - and a write with a new key waits for it to close. Retryable. + SchemaHandover { node: String }, /// An operator asked for something the cluster will not do, and the message says why. /// /// A string rather than a variant per cause: these are read by a person at a terminal, they @@ -115,6 +131,9 @@ impl ClusterError { Self::Unreachable { .. } | Self::LeaderUnreachable { .. } | Self::NotServing { .. } + | Self::NotSchemaLeader { .. } + | Self::SchemaLeaseLost { .. } + | Self::SchemaHandover { .. } | Self::StaleRoute { .. } | Self::RangeMoving { .. } => 503, Self::Peer { status, .. } => *status, @@ -150,6 +169,7 @@ impl ClusterError { self, Self::Unreachable { .. } | Self::NotServing { .. } + | Self::SchemaLeaseLost { .. } | Self::StaleRoute { .. } | Self::RangeMoving { .. } | Self::Timeout @@ -170,6 +190,9 @@ impl ClusterError { Self::Overflow => "sum_overflow", Self::Timeout => "query_timeout", Self::NotServing { .. } => "not_serving", + Self::NotSchemaLeader { .. } => "not_schema_leader", + Self::SchemaLeaseLost { .. } => "schema_lease_lost", + Self::SchemaHandover { .. } => "schema_handover", Self::Refused(_) => "refused", Self::StaleRoute { .. } => "stale_route", Self::RangeMoving { .. } => "range_moving", @@ -204,6 +227,21 @@ impl core::fmt::Display for ClusterError { } Self::Overflow => write!(f, "the totals from two nodes do not fit in one number"), Self::Timeout => write!(f, "the request ran out of time"), + Self::NotSchemaLeader { node, leader, epoch } => write!( + f, + "`{node}` does not lead the schema; as of epoch {epoch} that is {leader}. Ask \ + there instead" + ), + Self::SchemaLeaseLost { node } => write!( + f, + "`{node}` leads the schema and has lost touch with the agreement, so it cannot \ + know it still does; a key it has never assigned a row id waits until it can" + ), + Self::SchemaHandover { node } => write!( + f, + "`{node}` was just given the schema and has not finished taking it over; a key \ + nobody has assigned a row id waits until it has" + ), Self::NotServing { node, shards } => write!( f, "`{node}` holds shards {shards} and has lost touch with the agreement, so it \ diff --git a/crates/big-cluster/src/lib.rs b/crates/big-cluster/src/lib.rs index f717f1b..6f12275 100644 --- a/crates/big-cluster/src/lib.rs +++ b/crates/big-cluster/src/lib.rs @@ -71,7 +71,7 @@ mod admin; pub mod balance; -pub use admin::{MemberReport, MoveReport, RangeReport, Topology}; +pub use admin::{Balanced, MemberReport, MoveReport, RangeReport, Topology}; mod ddl; // The one item a sibling borrows across the split: `repair` recreates a field exactly as // another node has it, and that is a `Ddl` rather than a repair concern. @@ -148,9 +148,15 @@ use std::time::{Duration, Instant}; /// its data twice, so `Count` would silently double. That is precisely the quiet mistake a /// version exists to turn into a refusal. /// +/// `7` since a request to the schema leader says **who it thinks leads**. `InternRequest` and +/// `AllocateRequest` each gained an assumption at the end, and `/internal/schema/step-down` +/// was added. A `6` leader asked by a `7` coordinator would refuse the trailing bytes as +/// unreadable rather than intern for a coordinator it may no longer lead - the right failure, +/// and the version makes it a handshake failure instead of a per-request one. +/// /// The retired numbers are **not** reused. A stale peer that somehow got past the handshake /// would then misparse rather than fail, which is what `finished` exists to prevent. -pub const WIRE_VERSION: u32 = 6; +pub const WIRE_VERSION: u32 = 7; /// The header carrying [`WIRE_VERSION`]. pub const WIRE_HEADER: &str = "x-big-wire"; @@ -188,6 +194,12 @@ pub mod path { /// the row-key namespace changes hands. pub const FLOORS: &str = "/internal/floors"; pub const FLOORS_PUT: &str = "/internal/floors/put"; + /// Stop interning: the namespace is being handed over. Asked of the source before its + /// floor is read, so that the floor is final rather than racing. + pub const SCHEMA_STEP_DOWN: &str = "/internal/schema/step-down"; + /// Raise the agreement's ceiling on record ids for a table. Asked of the agreement's + /// leader by a schema leader that is not it. + pub const RESERVE: &str = "/internal/reserve"; } /// One node, playing whichever of the three roles a given request needs. @@ -219,6 +231,19 @@ pub struct Cluster { /// Not persisted, and does not need to be. A leader that restarts re-derives the first term /// from the data itself, and the floor only ever has to outlive the writes it is ahead of. allocated: std::sync::Mutex>, + /// One past the map epoch at which this node was last told to stop leading the schema. + /// Zero for never. + /// + /// A move of the namespace happens in steps, and between "stop" and the decision landing + /// this node still leads by the map and must not act like it - a source that went on + /// interning while its floor was being read would be the second interner the whole move + /// exists to prevent. Compared against the map's epoch, so leadership that comes back at + /// a later epoch is leadership again. + stood_down_at: std::sync::atomic::AtomicU64, + /// Whether the last attempt to hand the namespace to an elected successor found two + /// survivors disagreeing about what a row id means. Nobody interns while this is set; + /// `/metrics` is how somebody finds out. + handover_blocked: std::sync::atomic::AtomicBool, /// The agreement, when there is anything to agree about. /// /// `None` when no range has a copy: a range of one cannot fail over to anything, so @@ -324,6 +349,8 @@ impl Cluster

{ ranges, counters: counters::Counters::new(), allocated: Default::default(), + stood_down_at: Default::default(), + handover_blocked: Default::default(), }) } @@ -335,7 +362,9 @@ impl Cluster

{ None => (0, false), Some(c) => (c.term(), c.is_leader()), }; - let behind = self.map().stale.len(); + let map = self.map(); + let behind = map.stale.len(); + let moving = map.ranges.iter().filter(|r| r.moving.is_some()).count(); counters::Snapshot { nodes: self.config.nodes().len(), peers: self.config.nodes().len() - 1, @@ -344,6 +373,9 @@ impl Cluster

{ term, leader, behind, + moving, + schema_ready: map.schema_ready, + handover_blocked: self.handover_blocked.load(std::sync::atomic::Ordering::Relaxed), counts: self.counters.read(), } } diff --git a/crates/big-cluster/src/ownership.rs b/crates/big-cluster/src/ownership.rs index 849e1ec..d72b9db 100644 --- a/crates/big-cluster/src/ownership.rs +++ b/crates/big-cluster/src/ownership.rs @@ -75,11 +75,90 @@ impl Cluster

{ self.ranges.read().expect("no panic holds this lock").schema_leader } - /// Whether this node is the one that assigns row ids. + /// Whether the map names this node as the one that assigns row ids. + /// + /// The map's answer and nothing more. Whether this node may *act* on it is + /// [`Cluster::guard_schema`], which also asks whether it has stood down and whether it has + /// heard from the agreement recently enough to know it still leads. pub(super) fn leads_schema(&self) -> bool { self.schema_leader() == self.config.this_index() } + /// Who leads the schema and at which epoch the map said so, read together so that a + /// request built from them carries one consistent assumption. + pub(super) fn schema_lead(&self) -> (usize, u64) { + let map = self.ranges.read().expect("no panic holds this lock"); + (map.schema_leader, map.epoch) + } + + /// The refusal this node owes a request to intern or allocate when it may not. + /// + /// Three ways to owe one. **The map names somebody else**: a coordinator holding an older + /// map sent this here, and the answer says where to send it instead. **This node was told + /// to stand down** by a move that has not committed yet, so by the map it still leads and + /// by the move it no longer does. **This node has lost touch with the agreement**, so it + /// cannot know that it has not been replaced - the same reason a range stops being served, + /// and here there is no unfenced case: the schema leader can always be replaced. + /// + /// Two nodes interning at once hand one string two row ids, which nothing downstream can + /// see. Every refusal here is worth that. + pub fn guard_schema(&self) -> Result<()> { + let (leader, epoch) = self.schema_lead(); + let this = self.config.this_index(); + let node = self.config.this().name.clone(); + if leader != this { + return Err(ClusterError::NotSchemaLeader { + node, + leader: self.describe(leader), + epoch, + }); + } + if self.stood_down_at.load(std::sync::atomic::Ordering::Relaxed) > epoch { + return Err(ClusterError::NotSchemaLeader { + node, + leader: "a node this one is handing the namespace to".to_string(), + epoch, + }); + } + if !self.ranges.read().expect("no panic holds this lock").schema_ready { + return Err(ClusterError::SchemaHandover { node }); + } + if !self.controller.as_ref().is_none_or(|c| c.may_lead_schema()) { + return Err(ClusterError::SchemaLeaseLost { node }); + } + Ok(()) + } + + /// Refuses a request to the schema leader whose assumptions this node does not share. + /// + /// The one assumption a sender can state is who it thinks leads. A sender that names + /// another node reached this one by a map it has not yet applied - or by one this node has + /// not - and is told who leads *here*, so that it can ask the right node rather than have + /// this one answer a question that is no longer its to answer. + pub fn check_schema_lead(&self, led: Option<&wire::Led>) -> Result<()> { + if let Some(led) = led { + if led.leader != self.config.this_index() { + let (leader, epoch) = self.schema_lead(); + return Err(ClusterError::NotSchemaLeader { + node: self.config.this().name.clone(), + leader: self.describe(leader), + epoch, + }); + } + } + self.guard_schema() + } + + /// Stops this node interning until the map has moved past `epoch`. + /// + /// What a move of the namespace asks of its source before it reads the floor: after this + /// the floor is final rather than racing, and the window between here and the decision + /// landing is one in which nobody interns - which is allowed - rather than one in which + /// two nodes do. + pub fn stand_down_schema(&self, epoch: u64) { + self.stood_down_at.fetch_max(epoch.saturating_add(1), std::sync::atomic::Ordering::Relaxed); + } + /// A node's name, or `None` for an index the cluster file never had. /// /// An option rather than an index, because a node can now join at runtime: a `NodeId` from diff --git a/crates/big-cluster/src/raft.rs b/crates/big-cluster/src/raft.rs index 7833703..93d122a 100644 --- a/crates/big-cluster/src/raft.rs +++ b/crates/big-cluster/src/raft.rs @@ -124,6 +124,23 @@ pub struct RangeMap { /// The node that owns the row-key namespace. In the map rather than the file because /// draining the node holding it has to be able to move it. pub schema_leader: NodeId, + /// One past the highest record id the schema leader has been given leave to hand out, per + /// table. Sorted by table, so that two leaders building the same map build the same bytes. + /// + /// **The durable half of the allocation floor.** A leader hands out ids from a number in + /// its memory; a successor that started from what is on disk would hand out ids the old + /// leader had already promised to a write that had not landed - the one failure with no + /// second chance. So a leader commits a ceiling here before it hands out anything under + /// it, in blocks, and a successor starts at the ceiling. What a dead leader promised is + /// below it by construction. + pub reserved: Vec<(String, u64)>, + /// Whether the schema leader has finished taking the namespace over. + /// + /// `false` from the moment leadership moves until the successor has reconciled every row + /// key it could reach. While it is false nobody interns - not the old leader, which has + /// been deposed, and not the new one, which does not yet hold the mapping. A write with a + /// new key waits, which is allowed; two nodes interning at once is not. + pub schema_ready: bool, } /// Why a proposed map was refused. @@ -134,15 +151,44 @@ pub struct RangeMap { #[derive(Clone, PartialEq, Eq, Debug)] pub enum MapError { Empty, - NotStartingAtZero { start: ShardId }, - Unordered { at: usize }, - Overlap { from: ShardId, to: ShardId }, - Gap { from: ShardId, to: ShardId }, - NotTotal { from: ShardId }, - EmptyGroup { id: RangeId }, - PrimaryNotInGroup { id: RangeId, primary: NodeId }, - DuplicateId { id: RangeId }, - NoSuchRange { id: RangeId }, + NotStartingAtZero { + start: ShardId, + }, + Unordered { + at: usize, + }, + Overlap { + from: ShardId, + to: ShardId, + }, + Gap { + from: ShardId, + to: ShardId, + }, + NotTotal { + from: ShardId, + }, + EmptyGroup { + id: RangeId, + }, + PrimaryNotInGroup { + id: RangeId, + primary: NodeId, + }, + DuplicateId { + id: RangeId, + }, + NoSuchRange { + id: RangeId, + }, + /// The schema leader names a node the cluster does not have, or one that has left. + SchemaLeaderNotAMember { + node: NodeId, + }, + /// The schema leader names a node still catching up on the log. + SchemaLeaderIsALearner { + node: NodeId, + }, } impl core::fmt::Display for MapError { @@ -169,11 +215,58 @@ impl core::fmt::Display for MapError { } Self::DuplicateId { id } => write!(f, "two ranges are both called {id}"), Self::NoSuchRange { id } => write!(f, "there is no range {id}"), + Self::SchemaLeaderNotAMember { node } => write!( + f, + "the schema leader is node {node}, which is not in the cluster; nobody would \ + assign row ids" + ), + Self::SchemaLeaderIsALearner { node } => write!( + f, + "the schema leader is node {node}, which is still catching up on the log and \ + cannot be trusted with the row-key namespace yet" + ), } } } impl RangeMap { + /// One past the highest record id the schema leader may hand out for `table` without + /// asking the agreement first. Zero for a table nothing has been reserved for. + pub fn reserved_for(&self, table: &str) -> u64 { + self.reserved.iter().find(|(t, _)| t == table).map_or(0, |(_, upto)| *upto) + } + + /// Raises the reservation for `table` to `upto`, never lowering it. + /// + /// Never lowering is what makes it safe to apply twice, and what makes a block that was + /// reserved and never used harmless: a reservation is a promise about ids that *may* have + /// been handed out, and the higher of two promises keeps both. + pub fn reserve(&mut self, table: &str, upto: u64) { + match self.reserved.binary_search_by(|(t, _)| t.as_str().cmp(table)) { + Ok(i) => self.reserved[i].1 = self.reserved[i].1.max(upto), + Err(i) => self.reserved.insert(i, (table.to_string(), upto)), + } + } + + /// [`RangeMap::check`], and then whether the schema leader is a node that can hold the + /// namespace: a member that has not left, and not a learner. + /// + /// A second entry point because the map alone cannot tell: `schema_leader` is an index + /// into a membership the map does not carry. Every proposal goes through this one; + /// `check` is what a map can say about itself. + pub fn check_with(&self, members: &[Member]) -> core::result::Result<(), MapError> { + self.check()?; + let node = self.schema_leader; + match members.get(node) { + Some(m) if !m.reachable() => Err(MapError::SchemaLeaderNotAMember { node }), + Some(m) if matches!(m.state, MemberState::Learner) => { + Err(MapError::SchemaLeaderIsALearner { node }) + } + Some(_) => Ok(()), + None => Err(MapError::SchemaLeaderNotAMember { node }), + } + } + /// Whether this map covers the shard space exactly once, and every range is held. /// /// **Checked on every proposal rather than only at startup.** A file was read once and @@ -821,6 +914,11 @@ impl Raft { if !state.log.is_empty() { self.log = state.log; } + // The seed a snapshot handed over outranks the file's: empty is a file from before it + // was written down, and those nodes fell back to the file, which this still does. + if !state.seed.is_empty() { + self.seed = state.seed; + } // **Before the commit index is used**, because who is in the cluster is a property of // the log this node came back holding - not of the file it was first started with, // which may describe a cluster that no longer exists. @@ -842,6 +940,7 @@ impl Raft { commit: self.commit, base: self.base, log: self.log.clone(), + seed: self.seed.clone(), } } @@ -907,6 +1006,23 @@ impl Raft { .unwrap_or(0) .min(self.applied); let target = safe.saturating_sub(keep); + // **Never past the newest map or the newest membership.** The state machine is rebuilt + // by replaying the log, so the entry that last decided each has to stay in it. A + // follower handed a snapshot folds the state into `log[0]`; this is the leader's + // equivalent, and it costs at most two entries, for ever. Without it a leader that had + // compacted past its last decision came back from a restart with the file's map and + // the file's members - a cluster that, once anything had moved or joined, was gone. + let anchor = |want: fn(&Decision) -> bool| { + self.log.iter().rposition(|e| want(&e.decision)).map(|at| self.base + at as Index) + }; + let keep_from = [ + anchor(|d| matches!(d, Decision::Ranges(_))), + anchor(|d| matches!(d, Decision::Members(_))), + ] + .into_iter() + .flatten() + .min(); + let target = keep_from.map_or(target, |k| target.min(k)); if target <= self.base { return; } @@ -989,10 +1105,26 @@ impl Raft { self.heard.get(&node).copied() } + /// The highest index this node knows `node` has stored. **Leader's knowledge**: a follower + /// tracks nobody, so this is `None` everywhere but on the node that sends the appends. + pub fn matched(&self, node: NodeId) -> Option { + self.matched.get(&node).copied() + } + pub fn last_index(&self) -> Index { self.base + self.log.len() as Index - 1 } + /// Whether everything this node holds has been committed - the log has settled. + /// + /// Both sides are log positions. `commit` is one already; the last index is + /// `base + len - 1`, and the base is the part that was forgotten once, in a guard that + /// compared `commit` with the vector's length and refused every promotion after the first + /// compaction. + pub fn settled(&self) -> bool { + self.commit == self.last_index() + } + fn last_term(&self) -> Term { self.log.last().map_or(0, |e| e.term) } @@ -1484,6 +1616,14 @@ pub struct State { /// The index of `log[0]`. Zero until something has been compacted away. pub base: Index, pub log: Vec, + /// The cluster this node was started with, before the log said otherwise. + /// + /// Written down because a snapshot replaces it: a follower handed the state by its leader + /// takes the members that came with it as its seed, and a restart that forgot them would + /// fall back to the file - which, once anybody has joined or left, describes a cluster that + /// no longer exists. Empty in a file written before this was recorded, and empty means "the + /// file's", which is exactly what those nodes did. + pub seed: Vec, } pub trait Store: Send + Sync { @@ -1525,7 +1665,12 @@ impl FileStore { /// Bumped with the shape of the file. A `BIGRAFT1` file has a different header and different /// decision tags, and reading one as this format would take a count for a term. -const MAGIC: &[u8; 8] = b"BIGRAFT2"; +/// +/// `BIGRAFT3` appends the seed membership after the log. A `BIGRAFT2` file is the same bytes +/// without it, and is still read: a node upgraded in place has a file from before the seed was +/// recorded, and refusing it would be refusing the vote it already cast. +const MAGIC: &[u8; 8] = b"BIGRAFT3"; +const PREVIOUS: &[u8; 8] = b"BIGRAFT2"; impl Store for FileStore { fn load(&self) -> std::io::Result> { @@ -1662,9 +1807,20 @@ fn put_range_map(out: &mut Vec, m: &RangeMap) { for n in &m.stale { put_u64(out, *n as u64); } + put_u64(out, m.reserved.len() as u64); + for (table, upto) in &m.reserved { + put_str(out, table); + put_u64(out, *upto); + } + out.push(m.schema_ready as u8); } -fn get_range_map(b: &mut Bytes<'_>) -> core::result::Result { +/// `reservations` is whether the bytes carry the two fields `BIGRAFT3` added after `stale`. A +/// `BIGRAFT2` map has neither: nothing was reserved, and its leader was ready. +fn get_range_map( + b: &mut Bytes<'_>, + reservations: bool, +) -> core::result::Result { let epoch = b.u64()?; let schema_leader = b.u64()? as NodeId; let ranges = b.list(|b| { @@ -1682,7 +1838,12 @@ fn get_range_map(b: &mut Bytes<'_>) -> core::result::Result, members: &[Member]) { @@ -1739,13 +1900,16 @@ fn encode_state(state: &State) -> Vec { } } } + put_members(&mut out, &state.seed); out } fn decode_state(bytes: &[u8]) -> core::result::Result { - if bytes.get(..8) != Some(MAGIC) { - return Err("not a raft state file"); - } + let carries_seed = match bytes.get(..8) { + Some(m) if m == MAGIC => true, + Some(m) if m == PREVIOUS => false, + _ => return Err("not a raft state file"), + }; let mut b = Bytes { b: bytes, at: 8 }; let term = b.u64()?; let voted = b.u64()?; @@ -1756,16 +1920,17 @@ fn decode_state(bytes: &[u8]) -> core::result::Result { let term = b.u64()?; let decision = match b.byte()? { 0 => Decision::Noop, - 1 => Decision::Ranges(get_range_map(b)?), + 1 => Decision::Ranges(get_range_map(b, carries_seed)?), 2 => Decision::Members(get_members(b)?), _ => return Err("unknown decision"), }; Ok(Entry { term, decision }) })?; + let seed = if carries_seed { get_members(&mut b)? } else { Vec::new() }; if b.at != bytes.len() { return Err("bytes after the end"); } - Ok(State { term, voted_for, commit, base, log }) + Ok(State { term, voted_for, commit, base, log, seed }) } /// The members whose agreement counts, by index. diff --git a/crates/big-cluster/src/wire/agreement.rs b/crates/big-cluster/src/wire/agreement.rs index 8f1ca62..acd2d91 100644 --- a/crates/big-cluster/src/wire/agreement.rs +++ b/crates/big-cluster/src/wire/agreement.rs @@ -179,6 +179,12 @@ fn put_range_map(out: &mut Vec, m: &raft::RangeMap) { for n in &m.stale { put_u64(out, *n as u64); } + put_count(out, m.reserved.len()); + for (table, upto) in &m.reserved { + put_str(out, table); + put_u64(out, *upto); + } + put_bool(out, m.schema_ready); } fn get_range_map(r: &mut Reader<'_>) -> Result { @@ -221,7 +227,13 @@ fn get_range_map(r: &mut Reader<'_>) -> Result { for _ in 0..count { stale.push(r.u64()? as raft::NodeId); } - Ok(raft::RangeMap { epoch, ranges, stale, schema_leader }) + let count = r.count()?; + let mut reserved = Vec::with_capacity(count); + for _ in 0..count { + reserved.push((r.str()?, r.u64()?)); + } + let schema_ready = r.bool()?; + Ok(raft::RangeMap { epoch, ranges, stale, schema_leader, reserved, schema_ready }) } fn put_members(out: &mut Vec, members: &[raft::Member]) { diff --git a/crates/big-cluster/src/wire/request.rs b/crates/big-cluster/src/wire/request.rs index 15589f1..980c298 100644 --- a/crates/big-cluster/src/wire/request.rs +++ b/crates/big-cluster/src/wire/request.rs @@ -183,12 +183,45 @@ impl RecordsRequest { } } +/// What a request to the schema leader assumed about who leads, so that the receiver can +/// disagree with it. +/// +/// The routed envelope's shape, for the one role that is not a range. A coordinator holding a +/// map one decision old sends its keys to whoever led *then*; a receiver that no longer leads +/// answers with who does rather than interning anyway, which is the two-interners failure that +/// hands one string two row ids and nothing downstream can see. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Led { + /// The map epoch the sender read the leader from. + pub epoch: u64, + /// Who the sender believes leads the schema. + pub leader: usize, +} + +pub fn put_led(out: &mut Vec, led: Option<&Led>) { + put_bool(out, led.is_some()); + let Some(led) = led else { return }; + put_u64(out, led.epoch); + put_u64(out, led.leader as u64); +} + +pub fn get_led(r: &mut Reader<'_>) -> Result> { + if !r.bool()? { + return Ok(None); + } + let epoch = r.u64()?; + let leader = r.u64()? as usize; + Ok(Some(Led { epoch, leader })) +} + /// `POST /internal/intern`: what do these keys mean. Only the schema leader answers. #[derive(Clone, PartialEq, Eq, Debug)] pub struct InternRequest { pub table: String, pub field: String, pub keys: Vec, + /// Who the sender thinks leads. `None` from a caller that made no assumption. + pub led: Option, } impl InternRequest { @@ -200,6 +233,7 @@ impl InternRequest { for k in &self.keys { put_str(&mut out, k); } + put_led(&mut out, self.led.as_ref()); out } @@ -212,8 +246,9 @@ impl InternRequest { for _ in 0..n { keys.push(r.str()?); } + let led = get_led(&mut r)?; finished(&r)?; - Ok(Self { table, field, keys }) + Ok(Self { table, field, keys, led }) } } @@ -227,6 +262,8 @@ pub struct AllocateRequest { pub table: String, /// How many consecutive ids the caller needs. A whole statement asks once. pub count: u64, + /// Who the sender thinks leads. `None` from a caller that made no assumption. + pub led: Option, } impl AllocateRequest { @@ -234,6 +271,7 @@ impl AllocateRequest { let mut out = Vec::new(); put_str(&mut out, &self.table); put_u64(&mut out, self.count); + put_led(&mut out, self.led.as_ref()); out } @@ -241,8 +279,9 @@ impl AllocateRequest { let mut r = Reader::new(bytes); let table = r.str()?; let count = r.u64()?; + let led = get_led(&mut r)?; finished(&r)?; - Ok(Self { table, count }) + Ok(Self { table, count, led }) } } diff --git a/crates/big-cluster/src/write.rs b/crates/big-cluster/src/write.rs index a8de765..4f4bd54 100644 --- a/crates/big-cluster/src/write.rs +++ b/crates/big-cluster/src/write.rs @@ -28,6 +28,14 @@ use super::*; /// node that did not propose it. const STALE_ROUTE_PAUSE_MS: u64 = 500; +/// How many record ids the schema leader reserves with the agreement at a time. +/// +/// A ceiling committed per statement would put an agreement round trip in front of every +/// `INSERT`; a block this size puts one in front of every sixty-five thousand. What a block +/// costs is that a leader replaced mid-block leaves a gap of unused ids behind it - and record +/// ids are sixty-four bits wide and shard-mapped, so a gap costs nothing at all. +const RESERVE_BLOCK: u64 = 65_536; + impl Cluster

{ /// Writes a batch, splitting it by the owner of each record. /// @@ -297,11 +305,13 @@ impl Cluster

{ /// /// Answers the first id of the run; the caller takes `count` consecutive ids from it. pub(super) fn allocate(&self, table: &str, count: u64) -> Result { - if self.leads_schema() { + let (leader, epoch) = self.schema_lead(); + if leader == self.config.this_index() { + self.guard_schema()?; return self.allocate_here(table, count); } - let leader = self.schema_leader(); - let body = wire::AllocateRequest { table: table.to_string(), count }.encode(); + let led = Some(wire::Led { epoch, leader }); + let body = wire::AllocateRequest { table: table.to_string(), count, led }.encode(); let bytes = self.ask(leader, path::ALLOCATE, &body, None).map_err(|e| match e { ClusterError::Unreachable { node, why, .. } => { ClusterError::LeaderUnreachable { node, why } @@ -326,8 +336,21 @@ impl Cluster

{ // released before the floor was raised would be a lock that decided nothing. let mut floor = self.allocated.lock().unwrap_or_else(|e| e.into_inner()); let anywhere = self.next_record(table)?; - let from = anywhere.max(floor.get(table).copied().unwrap_or(0)); - floor.insert(table.to_string(), from.saturating_add(count)); + let ceiling = self.map().reserved_for(table); + // A leader with no floor of its own for this table is a fresh one - just elected, or + // just restarted - and starts at the ceiling: everything below it may have been + // promised by whoever led before, to a write that has not landed, and the number that + // would have said so died with them. + let promised = floor.get(table).copied().unwrap_or(ceiling); + let from = anywhere.max(promised); + let end = from.saturating_add(count); + if end > ceiling { + // **Committed before it is handed out.** A block, not the run: one entry in the + // agreement per `RESERVE_BLOCK` ids rather than per statement, and a block a dead + // leader never finished is a gap in the ids, which cost nothing. + self.reserve_ids(table, from.saturating_add(count.max(RESERVE_BLOCK)))?; + } + floor.insert(table.to_string(), end); Ok(from) } @@ -375,14 +398,19 @@ impl Cluster

{ /// Not queued, not assigned locally and reconciled later: two row ids for one string is a /// silently wrong answer, and a refusal is not. pub(super) fn intern(&self, table: &str, field: &str, keys: &[&str]) -> Result> { - if self.leads_schema() { + let (leader, epoch) = self.schema_lead(); + if leader == self.config.this_index() { + // Named by the map is not the same as allowed to act: a node that has stood down + // for a move, or has lost touch with the agreement, refuses here exactly as it + // would refuse a peer. + self.guard_schema()?; return Ok(self.api.intern_keys(table, field, keys)?); } - let leader = self.schema_leader(); let body = wire::InternRequest { table: table.to_string(), field: field.to_string(), keys: keys.iter().map(|k| (*k).to_string()).collect(), + led: Some(wire::Led { epoch, leader }), } .encode(); let bytes = self.ask(leader, path::INTERN, &body, None).map_err(|e| match e { diff --git a/crates/big-cluster/tests/balance.rs b/crates/big-cluster/tests/balance.rs index 939803a..5729f6b 100644 --- a/crates/big-cluster/tests/balance.rs +++ b/crates/big-cluster/tests/balance.rs @@ -35,17 +35,89 @@ fn range(id: u64, start: u64, end: Option, primary: usize) -> Range { } fn map(ranges: Vec) -> RangeMap { - RangeMap { epoch: 1, ranges, stale: Vec::new(), schema_leader: 0 } + RangeMap { + epoch: 1, + ranges, + stale: Vec::new(), + schema_leader: 0, + reserved: Vec::new(), + schema_ready: true, + } } +/// Every node that answered has also caught up on the log, which is the common case. fn load(pages: &[Option], frontier: u64) -> Vec { - pages.iter().map(|p| NodeLoad { pages: *p, frontier }).collect() + pages.iter().map(|p| NodeLoad { pages: *p, frontier, caught_up: p.is_some() }).collect() +} + +/// The same, with node `n` still replaying what it missed. +fn behind(mut loads: Vec, n: usize) -> Vec { + loads[n].caught_up = false; + loads } fn on() -> Policy { Policy { enabled: true, ..Policy::default() } } +// ------------------------------------------------------------------------------------------- +// Admitting a node that joined +// ------------------------------------------------------------------------------------------- + +/// **A learner that is answering and has caught up is made a member, before anything else.** +/// Until it is a voter the balancer cannot see it as a node with nothing, so admitting it has +/// to come first - even ahead of a drain an operator asked for, which only needs somewhere to +/// send the range and will have one more place the moment this lands. +#[test] +fn a_learner_that_answers_and_has_caught_up_is_admitted_before_anything_else() { + let m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 1)]); + let mut members = voters(&["a", "b"]); + members.push(member("d", MemberState::Learner)); + let l = load(&[Some(100_000), Some(100_000), Some(0)], 0); + assert_eq!(plan(&m, &members, &l, &on()), Some(Action::Admit { node: "d".to_string() })); + + // Ahead of the drain: `b` wants to go, and it will have `d` to go to once `d` counts. + members[1].state = MemberState::Draining; + assert_eq!( + plan(&m, &members, &l, &on()), + Some(Action::Admit { node: "d".to_string() }), + "a member is worth more than a move, and the move comes right after" + ); +} + +/// A learner that did not say what it weighs is a learner that may not be up for data at all. +/// Replicating the log to it proves nothing about whether it can serve a range. +#[test] +fn a_learner_that_did_not_answer_is_not_admitted() { + let m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 1)]); + let mut members = voters(&["a", "b"]); + members.push(member("d", MemberState::Learner)); + let l = load(&[Some(100_000), Some(100_000), None], 0); + assert_eq!(plan(&m, &members, &l, &on()), None); +} + +/// A learner still replaying the log would raise the bar for every election while being able +/// to help decide none - the exact reason a learner is not a voter in the first place. +#[test] +fn a_learner_still_catching_up_is_not_admitted() { + let m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 1)]); + let mut members = voters(&["a", "b"]); + members.push(member("d", MemberState::Learner)); + let l = behind(load(&[Some(100_000), Some(100_000), Some(0)], 0), 2); + assert_eq!(plan(&m, &members, &l, &on()), None); +} + +/// Admitting is a change to the map's membership, and the one-at-a-time rule covers it too. +#[test] +fn nothing_is_admitted_while_something_is_moving() { + let mut m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 1)]); + m.ranges[0].moving = Some(Move { target: 1, state: MoveState::Seeding }); + let mut members = voters(&["a", "b"]); + members.push(member("d", MemberState::Learner)); + let l = load(&[Some(100_000), Some(100_000), Some(0)], 0); + assert_eq!(plan(&m, &members, &l, &on()), None); +} + // ------------------------------------------------------------------------------------------- // When it does nothing // ------------------------------------------------------------------------------------------- @@ -137,13 +209,13 @@ fn a_draining_node_is_not_given_anything() { assert_eq!(plan(&m, &members, &l, &on()), None, "`b` holds nothing and still gets nothing"); } -/// A learner is still catching up: it replicates and does not yet take ranges. +/// A learner is still catching up: it replicates, is not yet admitted, and takes no range. #[test] fn a_learner_is_not_given_a_range_until_it_is_admitted() { let m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 0)]); let mut members = voters(&["a", "b"]); members[1].state = MemberState::Learner; - let l = load(&[Some(100_000), Some(0)], 70 * (1 << 20)); + let l = behind(load(&[Some(100_000), Some(0)], 70 * (1 << 20)), 1); assert_eq!(plan(&m, &members, &l, &on()), None); // Admitted, and now it is the obvious place for the tail. diff --git a/crates/big-cluster/tests/config.rs b/crates/big-cluster/tests/config.rs index be959ec..935544a 100644 --- a/crates/big-cluster/tests/config.rs +++ b/crates/big-cluster/tests/config.rs @@ -533,3 +533,29 @@ fn a_file_with_no_cluster_id_is_still_identified_by_its_shape() { assert_eq!(a.fingerprint(), b.fingerprint(), "which node this is does not change it"); assert_ne!(a.fingerprint(), edited.fingerprint(), "a file somebody edited does"); } + +/// **Two files that disagree about who leads the schema still describe one cluster.** +/// +/// The leader used to be folded into the shape, and that was right while it was a name read +/// once from a file. It moves at runtime now - by an operator, and by the agreement itself - +/// so a deployment whose files were updated at different moments after a *correct* failover +/// would have two nodes that could no longer talk: a successful failover causing a partition, +/// which is the worst failure on offer. The map overrides the file within a heartbeat, so +/// folding it in protected nothing. +#[test] +fn who_leads_the_schema_is_not_part_of_the_shape() { + let with = |leader: &str| { + format!( + "schema_leader = \"{leader}\"\n\ + [[node]]\nname = \"a\"\naddr = \"10.0.0.1:7654\"\nshards = \"0..64\"\n\ + [[node]]\nname = \"b\"\naddr = \"10.0.0.2:7654\"\nshards = \"64..\"\n" + ) + }; + let a = ClusterFile::parse(&with("a")).unwrap().for_node(Some("a"), "").unwrap(); + let b = ClusterFile::parse(&with("b")).unwrap().for_node(Some("a"), "").unwrap(); + assert_eq!(a.fingerprint(), b.fingerprint()); + + // The key is still required, and still has to name a node that could hold it: it seeds the + // map, and a fresh cluster has nothing else to start from. + assert!(matches!(ClusterFile::parse(&with("nobody")), Err(ConfigError::UnknownLeader { .. }))); +} diff --git a/crates/big-cluster/tests/peers.rs b/crates/big-cluster/tests/peers.rs index ab9ff2a..8f6c900 100644 --- a/crates/big-cluster/tests/peers.rs +++ b/crates/big-cluster/tests/peers.rs @@ -24,6 +24,7 @@ //! a real clock. Those stay in `big-http/tests/cluster.rs`, over real ports, where they belong. use big_cluster::client::{ClientError, PeerResponse, Peers, Repeatable}; +use big_cluster::raft::Store as _; use big_cluster::{ raft, wire, Cluster, ClusterConfig, ClusterError, ClusterFile, FactValue, OwnedFact, }; @@ -282,6 +283,46 @@ fn a_node_whose_vote_is_unreadable_refuses_to_start() { assert_eq!(e.kind(), std::io::ErrorKind::InvalidData, "{e}"); } +/// **A restart replays only what was committed, counted from the base.** +/// +/// The replay compared a position in the vector holding the log's suffix against the commit +/// index, which is a log position. Once anything had been compacted away, an entry past the +/// commit point looked committed, and a map no majority ever agreed to became this node's map. +#[test] +fn a_restart_applies_nothing_past_the_commit_point_once_the_log_has_a_base() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.raft"); + let config: ClusterConfig = ClusterFile::parse(THREE).unwrap().for_node(Some("a"), "").unwrap(); + let at = |epoch: u64| { + let mut m = config.seed_map(); + m.epoch = epoch; + raft::Decision::Ranges(m) + }; + let log = vec![ + // Index 40, the sentinel; 41, committed; 42, still in flight. + raft::Entry { term: 3, decision: at(2) }, + raft::Entry { term: 4, decision: at(3) }, + raft::Entry { term: 4, decision: at(4) }, + ]; + raft::FileStore::new(&path) + .save(&raft::State { term: 4, voted_for: Some(0), commit: 41, base: 40, log, seed: vec![] }) + .unwrap(); + + let api = Api::in_memory().unwrap(); + let peers = Fake::new(vec![None, Some(Reply::Unreachable), Some(Reply::Unreachable)]); + let c = Cluster::with_peers( + api, + config, + peers, + Box::new(raft::FileStore::new(&path)), + raft::Timing::default(), + Default::default(), + ) + .unwrap(); + assert_eq!(c.map().epoch, 3, "the committed map, not the one still in flight"); + c.stop(); +} + /// The other half of the same rule: **a file that is simply not there is a fresh node**, and a /// fresh node starts. Without this the fix above would be a cluster that cannot be deployed. #[test] diff --git a/crates/big-cluster/tests/raft.rs b/crates/big-cluster/tests/raft.rs index d29dc83..2a8f054 100644 --- a/crates/big-cluster/tests/raft.rs +++ b/crates/big-cluster/tests/raft.rs @@ -24,6 +24,7 @@ //! never lead the same term, and a committed decision is never taken back.** Everything else //! is a way of stressing that. +use big_cluster::controller::{Controller, Leases}; use big_cluster::raft::{ Decision, Member, MemberState, Message, NodeId, Raft, Range, RangeMap, Role, State, Store, Timing, @@ -172,7 +173,14 @@ fn map_of(primary: &[NodeId], stale: &[NodeId]) -> RangeMap { moving: None, }) .collect(); - RangeMap { epoch: 1, ranges, stale: stale.to_vec(), schema_leader: 0 } + RangeMap { + epoch: 1, + ranges, + stale: stale.to_vec(), + schema_leader: 0, + reserved: Vec::new(), + schema_ready: true, + } } /// A node alone in its config file leads immediately: a majority of one is one. @@ -537,11 +545,18 @@ fn a_restart_keeps_the_term_the_vote_and_the_log() { ]), }, ]; - let state = State { term: 4, voted_for: Some(2), commit: 3, base: 0, log: log.clone() }; + let state = State { + term: 4, + voted_for: Some(2), + commit: 3, + base: 0, + log: log.clone(), + seed: voters(3), + }; store.save(&state).unwrap(); let back = store.load().unwrap().expect("just written"); - assert_eq!(back, state); + assert_eq!(back, state, "the seed included: a snapshot replaces it, so it has to be kept"); // **The commit point is part of the state.** A node that forgot it would replay its whole // log into the state machine on the way up, applying a map no majority ever agreed to. @@ -549,7 +564,9 @@ fn a_restart_keeps_the_term_the_vote_and_the_log() { // A node with no vote is a different state from a node that voted for node zero, and the // two must not encode alike. - store.save(&State { term: 9, voted_for: None, commit: 1, base: 0, log }).unwrap(); + store + .save(&State { term: 9, voted_for: None, commit: 1, base: 0, log, seed: Vec::new() }) + .unwrap(); assert_eq!(store.load().unwrap().unwrap().voted_for, None); } @@ -562,7 +579,9 @@ fn a_restarted_node_still_refuses_a_stale_candidate() { big_cluster::raft::Entry { term: 0, decision: Decision::Noop }, big_cluster::raft::Entry { term: 7, decision: owning(&[1, 0]) }, ]; - store.save(&State { term: 7, voted_for: Some(1), commit: 1, base: 0, log }).unwrap(); + store + .save(&State { term: 7, voted_for: Some(1), commit: 1, base: 0, log, seed: Vec::new() }) + .unwrap(); let state = store.load().unwrap().unwrap(); let mut node = Raft::new(0, voters(3), Timing::default(), 0); @@ -591,6 +610,7 @@ fn a_damaged_state_file_is_refused() { commit: 0, base: 0, log: vec![big_cluster::raft::Entry { term: 0, decision: Decision::Noop }], + seed: Vec::new(), }) .unwrap(); @@ -769,7 +789,7 @@ fn a_restarted_node_comes_back_with_the_cluster_as_the_log_left_it() { // Started from a file that knows three nodes, restored from a log that knows four. let mut node = Raft::new(0, voters(3), Timing::default(), 0); assert_eq!(node.members(), 3); - node.restore(State { term: 5, voted_for: Some(0), commit: 1, base: 0, log }); + node.restore(State { term: 5, voted_for: Some(0), commit: 1, base: 0, log, seed: Vec::new() }); assert_eq!(node.members(), 4, "the log is what says who is in the cluster"); assert_eq!(node.voters(), &[0, 1, 2], "and the fourth is still catching up"); @@ -887,12 +907,17 @@ fn a_follower_that_falls_behind_the_base_is_caught_up_by_a_snapshot() { fn a_compacted_log_survives_a_restart() { let dir = tempfile::tempdir().unwrap(); let store = big_cluster::raft::FileStore::new(dir.path().join("state.raft")); + // With a reservation in it: the ceiling on record ids is the one thing in the map that a + // successor cannot rebuild from anywhere else, so it had better survive a restart. + let mut kept = map_of(&[1, 0], &[]); + kept.reserve("tx", 1 << 16); + kept.schema_ready = false; let log = vec![ // The sentinel is the state as of the base, not an empty entry. - big_cluster::raft::Entry { term: 3, decision: Decision::Ranges(map_of(&[1, 0], &[])) }, + big_cluster::raft::Entry { term: 3, decision: Decision::Ranges(kept) }, big_cluster::raft::Entry { term: 4, decision: Decision::Noop }, ]; - let state = State { term: 4, voted_for: Some(1), commit: 41, base: 40, log }; + let state = State { term: 4, voted_for: Some(1), commit: 41, base: 40, log, seed: voters(3) }; store.save(&state).unwrap(); assert_eq!(store.load().unwrap().unwrap(), state, "the base is part of what is written down"); @@ -902,3 +927,246 @@ fn a_compacted_log_survives_a_restart() { assert_eq!(node.last_index(), 41, "indices are log positions, not vector positions"); assert_eq!(node.commit_index(), 41); } + +/// **A restart lands on the seed the snapshot handed over**, not on the file's. +/// +/// A follower that fell behind the base is given the state whole, members included, and takes +/// those members as its seed. Until the seed was written down, a restart forgot that and went +/// back to the file - which, once anybody had joined or left, named a cluster that was gone. +#[test] +fn a_restored_seed_is_the_membership_when_the_log_says_nothing() { + let mut grown = voters(3); + grown.push(learner("joined")); + let log = vec![big_cluster::raft::Entry { term: 3, decision: Decision::Noop }]; + + let mut node = Raft::new(0, voters(3), Timing::default(), 0); + node.restore(State { term: 3, voted_for: None, commit: 40, base: 40, log, seed: grown }); + assert_eq!(node.members(), 4, "the seed that was written down is the one that counts"); + assert_eq!(node.voters(), &[0, 1, 2], "and the fourth is a learner, as the seed said"); +} + +/// A state file from before the seed was recorded still loads, and reads as "the file's". +#[test] +fn a_state_file_from_before_the_seed_was_recorded_still_loads() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.raft"); + // `BIGRAFT2`, by hand: magic, term, vote, commit, base, then a log of one `Noop`. + let mut bytes = b"BIGRAFT2".to_vec(); + for v in [7u64, u64::MAX, 0, 0, 1, 0] { + bytes.extend_from_slice(&v.to_le_bytes()); + } + bytes.push(0); + std::fs::write(&path, &bytes).unwrap(); + + let state = big_cluster::raft::FileStore::new(&path) + .load() + .unwrap() + .expect("a file from the previous format is still a file"); + assert_eq!(state.term, 7); + assert_eq!(state.voted_for, None); + assert_eq!(state.log.len(), 1); + assert!(state.seed.is_empty(), "nothing was written, so nothing is claimed"); +} + +/// **Compaction keeps the newest map and the newest membership**, whatever else it drops. +/// +/// The follower's snapshot path folds the state into `log[0]`; the leader's own compaction did +/// not, so a leader that compacted past its last decision and then restarted came back with the +/// file's map and the file's members - a cluster that, once anything had moved or joined, no +/// longer existed. +#[test] +fn a_compacted_log_still_holds_the_map_and_the_membership() { + let mut sim = Sim::new(4); + sim.run(3_000); + let leader = sim.leader(); + // The fourth node steps back to learning: still replicated to, no longer voting. + let mut shrunk = voters(4); + shrunk[3].state = MemberState::Learner; + + assert!(sim.propose(leader, owning(&[1, 2, 0]))); + sim.run(500); + assert!(sim.propose(leader, Decision::Members(shrunk))); + sim.run(500); + for _ in 0..(Raft::KEEP_ENTRIES + 8) { + assert!(sim.propose(leader, Decision::Noop)); + sim.run(100); + } + sim.nodes[leader].compact(0); + assert!(sim.nodes[leader].base() > 0, "there was something to compact"); + + let log = sim.nodes[leader].log(); + assert!( + log.iter().any(|e| matches!(e.decision, Decision::Ranges(_))), + "the newest map is still in the log" + ); + assert!( + log.iter().any(|e| matches!(e.decision, Decision::Members(_))), + "and so is the newest membership" + ); + + // And a restart from that log lands on both, whatever the file says. + let mut back = Raft::new(leader, voters(4), Timing::default(), 0); + back.restore(sim.nodes[leader].state()); + assert_eq!(back.members(), 4); + assert_eq!(back.voters(), &[0, 1, 2], "the membership survived the compaction and the restart"); +} + +/// **A compacted leader still fails a range over.** +/// +/// The guard in front of a promotion compared the commit index - a log position - with the +/// length of the vector holding the log's suffix. Once anything had been compacted away the +/// two could never agree again, and every promotion after the first compaction was silently +/// refused on a cluster reporting itself healthy. +#[test] +fn a_compacted_leader_still_fails_a_range_over() { + let mut sim = Sim::new(3); + sim.run(3_000); + let leader = sim.leader(); + for i in 0..8 { + assert!(sim.propose(leader, owning(&[i % 3, (i + 1) % 3]))); + sim.run(200); + } + sim.nodes[leader].compact(0); + assert!(sim.nodes[leader].base() > 0, "the log was compacted"); + + // One copy of a replicated range goes quiet for longer than the lease allows. + let dead = (leader + 1) % 3; + sim.down.insert(dead); + let leases = Leases::default(); + sim.run(leases.promote_after.as_millis() as u64 * 3); + assert_eq!(sim.leader(), leader, "two of three are still a majority"); + + let mut current = map_of(&[dead], &[]); + current.ranges[0].group = vec![dead, leader]; + let next = Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now) + .expect("a range whose primary went quiet is given to the copy that is answering"); + assert_eq!(next.ranges[0].primary, leader); + assert!(next.is_stale(dead), "and the one that went quiet is marked behind"); +} + +// ------------------------------------------------------------------------------------------- +// The row-key namespace, moved by the agreement +// +// The most expensive decision the agreement makes: it copies every row key of every table to +// the successor. So it waits far longer than a range failover, it happens only when an operator +// has said it may, and the node it takes the namespace from is marked behind whether or not it +// holds a copy of anything - it may have interned row ids nobody else ever saw. +// ------------------------------------------------------------------------------------------- + +/// A cluster of `n` in which the range is served by the agreement's leader, so that nothing +/// about the *ranges* moves and what is left to observe is the namespace alone. +fn quiet_schema_leader(n: usize) -> (Sim, NodeId, NodeId) { + let mut sim = Sim::new(n); + sim.run(3_000); + let leader = sim.leader(); + let holder = (leader + 1) % n; + sim.down.insert(holder); + (sim, leader, holder) +} + +fn schema_after(ms: u64) -> Leases { + Leases { move_schema_after: Some(std::time::Duration::from_millis(ms)), ..Leases::default() } +} + +/// **Silence long enough for a range is not long enough for the namespace.** Moving it copies +/// every row key of every table, so it waits until the ranges have already failed over and the +/// map has settled - and on a blip it does nothing at all. +#[test] +fn the_namespace_waits_far_longer_than_a_range_does() { + let (mut sim, leader, holder) = quiet_schema_leader(3); + let leases = schema_after(15_000); + let mut current = map_of(&[leader], &[]); + current.schema_leader = holder; + + // Past `promote_after`, nowhere near the namespace's own wait. + sim.run(6_000); + assert!( + Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now).is_none(), + "a range would have moved by now; the namespace has not" + ); + + sim.run(12_000); + let next = Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now) + .expect("silence for the whole of the longer wait moves it"); + assert_eq!(next.schema_leader, leader); + assert!(!next.schema_ready, "the successor holds no row keys yet, and says so"); + assert!(next.is_stale(holder), "the deposed node is marked behind, holding a copy or not"); +} + +/// **Off unless an operator turned it on.** The namespace stays where the file put it, however +/// long its holder has been gone - which is what every deployment before this did. +#[test] +fn the_namespace_is_never_moved_unless_a_lease_says_it_may() { + let (mut sim, leader, holder) = quiet_schema_leader(3); + let mut current = map_of(&[leader], &[]); + current.schema_leader = holder; + sim.run(60_000); + assert_eq!( + Controller::promotion_for(&Leases::default(), &sim.nodes[leader], ¤t, sim.now), + None + ); +} + +/// The successor is a voter that is answering and has not missed a write - the same three +/// conditions a range's replacement has to meet, for the same reasons. +#[test] +fn a_learner_a_draining_node_and_a_copy_that_is_behind_are_all_passed_over() { + let mut sim = + Sim::with_timing(4, Timing { election_min: 1_000, election_spread: 1_000, heartbeat: 200 }); + sim.run(3_000); + let leader = sim.leader(); + // Everybody but the leader is unfit in some way: one is a learner, one is behind, and the + // fourth is the node being deposed. + let holder = (leader + 1) % 4; + let learner_at = (leader + 2) % 4; + let behind_at = (leader + 3) % 4; + let mut members = voters(4); + members[learner_at].state = MemberState::Learner; + assert!(sim.propose(leader, Decision::Members(members))); + sim.run(500); + + sim.down.insert(holder); + sim.run(20_000); + + let mut current = map_of(&[leader], &[behind_at]); + current.schema_leader = holder; + let next = + Controller::promotion_for(&schema_after(15_000), &sim.nodes[leader], ¤t, sim.now) + .expect("there is one node left that is fit to hold it"); + assert_eq!(next.schema_leader, leader, "not the learner, not the one marked behind"); +} + +/// Two leaders elected in sequence decide the same thing from the same facts, which is what +/// keeps the namespace from moving twice. +#[test] +fn the_same_facts_always_choose_the_same_successor() { + let (mut sim, leader, holder) = quiet_schema_leader(3); + sim.run(20_000); + let mut current = map_of(&[leader], &[]); + current.schema_leader = holder; + let leases = schema_after(15_000); + + let once = Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now); + for _ in 0..20 { + assert_eq!(Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now), once); + } + assert!(once.is_some()); +} + +/// Nothing is decided while the namespace is still being handed over: the successor has not +/// finished taking it, and choosing another one would throw away the keys it already holds. +#[test] +fn a_handover_that_has_not_finished_is_not_restarted() { + let (mut sim, leader, holder) = quiet_schema_leader(3); + sim.run(20_000); + // Already deposed, already waiting on its successor. + let mut current = map_of(&[leader], &[holder]); + current.schema_leader = leader; + current.schema_ready = false; + + assert_eq!( + Controller::promotion_for(&schema_after(15_000), &sim.nodes[leader], ¤t, sim.now), + None, + "the node it names is this one, and it is answering; there is nothing to decide" + ); +} diff --git a/crates/big-cluster/tests/rangemap.rs b/crates/big-cluster/tests/rangemap.rs index f56606f..ee87241 100644 --- a/crates/big-cluster/tests/rangemap.rs +++ b/crates/big-cluster/tests/rangemap.rs @@ -21,7 +21,8 @@ use big_cluster::config::ClusterFile; use big_cluster::raft::{ - MapError, MergeError, Move, MoveError, MoveState, Range, RangeMap, SplitError, + MapError, Member, MemberState, MergeError, Move, MoveError, MoveState, Range, RangeMap, + SplitError, }; use big_engine::ShardRange; use proptest::prelude::*; @@ -31,7 +32,69 @@ fn range(id: u64, start: u64, end: Option, primary: usize) -> Range { } fn map(ranges: Vec) -> RangeMap { - RangeMap { epoch: 0, ranges, stale: Vec::new(), schema_leader: 0 } + RangeMap { + epoch: 0, + ranges, + stale: Vec::new(), + schema_leader: 0, + reserved: Vec::new(), + schema_ready: true, + } +} + +// ------------------------------------------------------------------------------------------- +// What the schema leader may hand out +// ------------------------------------------------------------------------------------------- + +/// **A reservation only ever rises**, and a table nothing was reserved for is at zero. Rising +/// only is what makes a reservation safe to apply twice and a block nobody used harmless: it +/// is a promise about ids that *may* have been handed out, and the higher promise keeps both. +#[test] +fn a_reservation_only_ever_rises_and_stays_sorted_by_table() { + let mut m = map(vec![range(0, 0, None, 0)]); + assert_eq!(m.reserved_for("tx"), 0, "nothing reserved is a ceiling of nothing"); + + m.reserve("tx", 65_536); + m.reserve("aa", 10); + m.reserve("tx", 100); + assert_eq!(m.reserved_for("tx"), 65_536, "a lower reservation changes nothing"); + assert_eq!(m.reserved_for("aa"), 10); + let tables: Vec<&str> = m.reserved.iter().map(|(t, _)| t.as_str()).collect(); + assert_eq!(tables, ["aa", "tx"], "sorted, so two leaders build the same bytes"); + + m.reserve("tx", 131_072); + assert_eq!(m.reserved_for("tx"), 131_072); +} + +/// **The schema leader has to be a node that can hold the namespace.** The map alone cannot +/// tell - it holds an index into a membership it does not carry - so every proposal is checked +/// against both, and a map naming a node that left or one still catching up is refused before +/// it can commit and leave nobody assigning row ids. +#[test] +fn a_schema_leader_that_left_or_is_still_learning_is_refused() { + let member = |name: &str, state| Member { + name: name.to_string(), + addr: format!("10.0.0.1:{}", name.len()), + state, + }; + let mut m = map(vec![range(0, 0, None, 0)]); + let members = vec![ + member("a", MemberState::Voter), + member("b", MemberState::Learner), + member("c", MemberState::Gone), + ]; + assert_eq!(m.check_with(&members), Ok(()), "a voter may lead"); + + m.schema_leader = 1; + assert_eq!(m.check_with(&members), Err(MapError::SchemaLeaderIsALearner { node: 1 })); + m.schema_leader = 2; + assert_eq!(m.check_with(&members), Err(MapError::SchemaLeaderNotAMember { node: 2 })); + m.schema_leader = 7; + assert_eq!(m.check_with(&members), Err(MapError::SchemaLeaderNotAMember { node: 7 })); + + // And the space is still checked first: a gap is a gap whoever leads. + let gappy = map(vec![range(0, 0, Some(8), 0), range(1, 16, None, 0)]); + assert_eq!(gappy.check_with(&members), Err(MapError::Gap { from: 8, to: 16 })); } // ------------------------------------------------------------------------------------------- diff --git a/crates/big-cluster/tests/wire.rs b/crates/big-cluster/tests/wire.rs index c87668a..0286aa5 100644 --- a/crates/big-cluster/tests/wire.rs +++ b/crates/big-cluster/tests/wire.rs @@ -402,6 +402,22 @@ proptest! { } } +/// A request to the schema leader carries who the sender thinks leads, so that the receiver can +/// disagree - and a caller with no assumption says so rather than inventing one. +#[test] +fn a_request_to_the_schema_leader_carries_who_it_thinks_leads() { + let intern = wire::InternRequest { + table: "t".to_string(), + field: "f".to_string(), + keys: vec!["GB".to_string(), "FR".to_string()], + led: Some(wire::Led { epoch: 9, leader: 2 }), + }; + assert_eq!(wire::InternRequest::decode(&intern.encode()).unwrap(), intern); + + let allocate = wire::AllocateRequest { table: "t".to_string(), count: 4, led: None }; + assert_eq!(wire::AllocateRequest::decode(&allocate.encode()).unwrap(), allocate); +} + /// A segment travels as what its records *hold*, not as encoded blocks. /// /// Shipping the encoded form would tie two nodes to the same codec choice for ever; shipping the diff --git a/crates/big-db/src/db/mod.rs b/crates/big-db/src/db/mod.rs index bd066d3..e8e2d5b 100644 --- a/crates/big-db/src/db/mod.rs +++ b/crates/big-db/src/db/mod.rs @@ -74,6 +74,11 @@ impl Db

{ Ok(self.store.set_durability(next)?) } + /// Gives trailing free pages back to the filesystem. See [`Store::truncate_tail`]. + pub fn reclaim(&self) -> Result { + Ok(self.store.truncate_tail()?) + } + pub fn store(&self) -> &Store

{ &self.store } @@ -123,6 +128,52 @@ impl Db

{ Ok(total) } + /// Counts the pages this file contains that nothing points at and nothing has recorded as + /// free. + /// + /// **It changes nothing.** A page like that is space the file will never use and never + /// give back, and - because [`Db::reclaim`] can only release pages flush against the end - + /// one of them at the top pins every free page beneath it. This says how many there are; + /// it does not free them, and the page numbers it reports must not be fed to anything that + /// does. See [`big_pager::LeakReport::leaked_sample`]. + /// + /// **Cost is a full read of the live data**, the same as [`Db::scrub`] or a backup, and it + /// holds a read transaction throughout - so the reclaim horizon does not move while it + /// runs. Not for a request path and not for a timer. + /// + /// This is not a scrub: the walk parses every page it reads but does not verify checksums, + /// except on the chains, where an unverified `next` would send the walk to an arbitrary + /// page and count it as live. Run `big scrub` for the checksums. + /// + /// Any error aborts the whole audit rather than skipping a tree. A mark phase that + /// swallowed an error would under-mark, and under-marking reports leaks that are not + /// there - a wrong number presented as a fact is worse than a failure. + pub fn audit_pages(&self) -> Result { + let mut audit = self.store.begin_audit()?; + + // The current trees. `visit_tree` yields branch pages, leaf pages and every page a + // leaf cell owns - dense bitmaps, the base page of a delta, and a columnar segment's + // spill page. Columnar segments share the root-record namespace, so they are already + // in `roots` and need no separate enumeration. + for root in core::mem::take(&mut audit.roots) { + big_btree::visit_tree(self.store.pager(), root, &mut |pgno, _| { + audit.mark_tree_page(pgno, false); + Ok(()) + })?; + } + + // **And the trees only a snapshot still names.** These are unreachable from the + // current roots and entirely live; `scrub` and `copy_to` both walk past them. + for root in core::mem::take(&mut audit.snapshot_roots) { + big_btree::visit_tree(self.store.pager(), root, &mut |pgno, _| { + audit.mark_tree_page(pgno, true); + Ok(()) + })?; + } + + Ok(audit.finish()) + } + /// What the row-key dictionary costs this process right now. /// /// Every row key is resident, in both directions, for the life of the process: a key has diff --git a/crates/big-db/tests/leaks.rs b/crates/big-db/tests/leaks.rs new file mode 100644 index 0000000..2e6e3c5 --- /dev/null +++ b/crates/big-db/tests/leaks.rs @@ -0,0 +1,225 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Every page of a real database accounted for. +//! +//! `crates/big-pager/tests/audit.rs` proves the counting; this proves the *walk* - that the +//! mark phase reaches every class of page a `Db` actually produces. A class it misses is a +//! class it reports as leaked, so each test below is one page class that would otherwise be +//! silently wrong. + +use big_db::*; +use big_pager::MemPager; + +fn audit(d: &Db) -> big_pager::LeakReport { + d.audit_pages().expect("the file is readable") +} + +/// Fragments spread one per shard: what fills a file is fragments, not records. +fn seeded(shards: u64) -> Db { + let d = Db::in_memory().unwrap(); + d.create_table("tx").unwrap(); + d.create_field("tx", "amount", FieldKind::Int, 32).unwrap(); + d.create_field("tx", "country", FieldKind::Set, 0).unwrap(); + let mut w = d.write(); + for i in 0..shards { + let record = i << 20; + w.set_int("tx", "amount", record, i * 7).unwrap(); + w.set_key("tx", "country", record, if i.is_multiple_of(3) { "vn" } else { "jp" }).unwrap(); + } + w.commit().unwrap(); + d +} + +/// **The number that must stay zero.** Everything below is one way of making it non-zero. +#[test] +fn a_fresh_database_accounts_for_every_page() { + let d = Db::in_memory().unwrap(); + let r = audit(&d); + assert!(r.is_clean(), "{r:?}"); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +#[test] +fn a_database_with_data_accounts_for_every_page() { + let d = seeded(200); + let r = audit(&d); + assert_eq!(r.leaked, 0, "{r:?}"); + assert_eq!(r.dangling, 0, "{r:?}"); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +/// **A columnar table's spill pages.** A leaf cell of kind `ValuesPtr` owns a page the way a +/// dense bitmap cell does; a walk that only followed bitmaps would report every one of them as +/// leaked, which is the loudest possible failure and therefore the best test. +#[test] +fn a_columnar_table_leaks_no_spill_pages() { + let d = Db::in_memory().unwrap(); + d.create_table_with("tx", TableEngine::Columnar).unwrap(); + d.create_field("tx", "amount", FieldKind::Int, 32).unwrap(); + let mut w = d.write(); + // Enough distinct values in one shard that the column spills to a page of its own. + for r in 0..5_000u64 { + w.set_int("tx", "amount", r, r * 31).unwrap(); + } + w.commit().unwrap(); + + let r = audit(&d); + assert_eq!(r.leaked, 0, "a columnar spill page was counted as leaked: {r:?}"); + assert!(r.by_class.trees > 0, "{r:?}"); +} + +/// **A delta cell's base page.** Writing a fragment twice leaves a cell that names the page +/// holding what it is a delta against. `visit_tree` follows it; a hand-rolled walk forgets it. +#[test] +fn a_fragment_written_twice_leaks_no_delta_base() { + let d = seeded(64); + let mut w = d.write(); + for i in 0..64u64 { + w.set_int("tx", "amount", (i << 20) + 1, i * 13).unwrap(); + } + w.commit().unwrap(); + + let r = audit(&d); + assert_eq!(r.leaked, 0, "{r:?}"); +} + +/// A database that has been mostly emptied still accounts for every page: what the delete +/// freed is in the freelist, and what is left is reachable. +#[test] +fn a_database_that_has_churned_leaks_nothing() { + let d = seeded(900); + let doomed: Vec = (0..900u64).filter(|i| !i.is_multiple_of(50)).map(|i| i << 20).collect(); + let mut w = d.write(); + w.delete("tx", &doomed).unwrap(); + w.commit().unwrap(); + let mut w = d.write(); + w.set_int("tx", "amount", 1, 1).unwrap(); + w.commit().unwrap(); + + let r = audit(&d); + assert_eq!( + r.leaked, 0, + "pages nothing points at and nothing recorded as free; highest is {:?}: {r:?}", + r.highest_leaked + ); +} + +/// **What `truncate_tail` stops at is always a page somebody owns.** +/// +/// It walks down from the end of the file and halts at the first page that is not a reusable +/// free run, so whatever sits on top pins every free page beneath it - on this fixture, thousands +/// of them. Before `Freelist::absorb_scratch` the page on top was routinely one that *nobody* +/// owned, and there was no way to give it back short of rewriting the whole file. +/// +/// This asserts the part that is now structural: reclaim may still stall, but never on a page the +/// file has lost track of. It deliberately does not assert how much comes back, because that is +/// not yet a property of the engine - see below. +/// +/// **What still pins the tail, measured on this fixture and in this order.** Neither is a leak; +/// the audit is right to report zero for both. +/// +/// 1. *The freelist chain of a large delete.* A transaction cannot reuse the pages it is itself +/// freeing - they carry its own id, which is above the horizon until it has committed - so the +/// commit that frees thousands of pages must put its own freelist chain at the top of the file. +/// An unchanged chain is then kept where it lies, so empty commits never move it; one real +/// write does, and the tail drops by the size of the chain. +/// 2. *A single live b-tree page.* Whatever is highest after that pins everything below it for +/// good. Moving it means rewriting whoever points at it, which is knowledge `big-btree` has and +/// `big-pager` does not - so `truncate_tail` cannot, by construction, and this is the work that +/// would make reclaim give back a churned file. +#[test] +fn reclaim_never_stalls_on_a_page_nobody_owns() { + let d = seeded(900); + let doomed: Vec = (0..900u64).filter(|i| !i.is_multiple_of(50)).map(|i| i << 20).collect(); + let mut w = d.write(); + w.delete("tx", &doomed).unwrap(); + w.commit().unwrap(); + let mut w = d.write(); + w.set_int("tx", "amount", 1, 1).unwrap(); + w.commit().unwrap(); + + d.store().truncate_tail().expect("no readers are alive here"); + + let r = audit(&d); + assert_eq!(r.leaked, 0, "reclaim stopped at a page nothing owns: {r:?}"); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +/// **The check nothing else in the tree makes.** A page both reachable and reusable-free has +/// been handed out twice. Reachable-and-*pending*-free is ordinary - that is what a snapshot +/// pinning an old tree looks like - so only the reusable half is evidence. +#[test] +fn no_page_is_both_live_and_free_to_reuse() { + let d = seeded(300); + let mut w = d.write(); + w.delete("tx", &(0..150u64).map(|i| i << 20).collect::>()).unwrap(); + w.commit().unwrap(); + + let r = audit(&d); + assert_eq!(r.double_allocated, 0, "a page was handed out twice: {r:?}"); + assert!(r.double_allocated_sample.is_empty(), "{r:?}"); +} + +/// The audit is exact as of the transaction it opened at, and stays right while writers commit +/// underneath it. This is the test that catches a scaffold captured across two different +/// states, and it is the only one that would. +#[test] +fn the_audit_is_stable_while_a_writer_commits() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + let d = Arc::new(seeded(120)); + let stop = Arc::new(AtomicBool::new(false)); + + let writer = { + let d = Arc::clone(&d); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + let mut n = 0u64; + while !stop.load(Ordering::Relaxed) { + let mut w = d.write(); + w.set_int("tx", "amount", (n % 120) << 20, n).unwrap(); + w.commit().unwrap(); + n += 1; + } + n + }) + }; + + for _ in 0..12 { + let r = audit(&d); + assert_eq!(r.leaked, 0, "{r:?}"); + assert_eq!(r.dangling, 0, "{r:?}"); + assert_eq!(r.double_allocated, 0, "{r:?}"); + } + stop.store(true, Ordering::Relaxed); + let commits = writer.join().unwrap(); + assert!(commits > 0, "the writer never got a turn, so this proved nothing"); +} + +/// A compact copy is a fresh history: it must account for every page it has. +#[test] +fn a_compacted_copy_accounts_for_every_page() { + let src = seeded(200); + let mut w = src.write(); + w.delete("tx", &(0..100u64).map(|i| i << 20).collect::>()).unwrap(); + w.commit().unwrap(); + + let copy = src.copy_to(MemPager::new()).unwrap(); + let r = audit(©); + assert_eq!(r.leaked, 0, "{r:?}"); + assert_eq!(r.free_total, 0, "a fresh copy has nothing free: {r:?}"); + assert_eq!(r.reachable, r.page_count, "{r:?}"); +} diff --git a/crates/big-embed/src/lib.rs b/crates/big-embed/src/lib.rs index 5660ef3..a592804 100644 --- a/crates/big-embed/src/lib.rs +++ b/crates/big-embed/src/lib.rs @@ -1416,6 +1416,26 @@ impl Api

{ self.db.store().metrics() } + /// Gives trailing free pages back to the filesystem, and answers with how many. + /// + /// **The only thing in the tree that makes a served database smaller.** Copy-on-write + /// leaves holes and the freelist reuses them, so a file that has churned is mostly free + /// space that is never handed back; `big compact` gives it back and wants the exclusive + /// lock, which means stopping the daemon. This does it in place, while serving. + /// + /// **Only pages flush against the end of the file**, because moving a page anywhere else + /// means rewriting whoever points at it. What makes the tail empty out on its own is that + /// the freelist hands out its lowest page first: ordinary writes fill the holes near the + /// front, and the end of the file drains. + /// + /// [`ApiError`] `ReadersActive` when a read transaction is open anywhere in this process - + /// shrinking the file turns the region past the new end back into unbacked mapping, and a + /// borrow into it would be a `SIGBUS`. It is an ordinary outcome rather than a failure: + /// try again when the node is between requests. + pub fn reclaim(&self) -> Result { + Ok(self.db.reclaim()?) + } + /// What the row-key dictionary costs this process. See [`Db::key_stats`]. pub fn key_stats(&self) -> KeyStats { self.db.key_stats() diff --git a/crates/big-http/src/lib.rs b/crates/big-http/src/lib.rs index d56dcbf..ab73eb6 100644 --- a/crates/big-http/src/lib.rs +++ b/crates/big-http/src/lib.rs @@ -47,6 +47,7 @@ pub mod json; pub mod metrics; pub mod routes; pub mod status; +mod steward; mod watchdog; /// Re-exported from [`big_wire`], which is where the HTTP/1.1 framing lives now. @@ -112,6 +113,13 @@ pub struct ServerConfig { /// /// When the cluster may reshape itself, and how hard. Off by default. pub balance: big_cluster::balance::Policy, + /// Whether this node may give trailing free pages back to the filesystem while it serves. + /// + /// Off by default, like everything else that acts without being asked. What it changes is + /// that a file which has churned can get smaller without stopping the daemon; what it + /// costs is the write lock for the length of one truncation, and only once a quarter of + /// the file is reclaimable. + pub reclaim: bool, /// An ordinary `Option` with no `#[cfg]` on it: `TlsConfig` is uninhabited in a build with /// the feature off, so this is provably `None` there and every construction site in the tree /// - tests included - compiles either way without knowing which build it is in. @@ -145,6 +153,7 @@ impl Default for ServerConfig { // server is the default, and `big serve` refuses to bind anywhere else without one. tls: None, balance: big_cluster::balance::Policy::default(), + reclaim: false, // Off. A daemon that backed itself up somewhere by default would be a daemon // filling a disk nobody chose. backup_dir: None, @@ -316,6 +325,18 @@ impl Server

{ }) }; + // The node's own slow work - balancing, and whatever else is switched on - on a thread + // that is neither the agreement's nor a worker's. See `steward.rs` for why it can be + // neither. Stopped the way the workers are: `stopping` is set on every way out of this + // loop, and it is joined after them. + let steward = { + let state = Arc::clone(&self.state); + std::thread::Builder::new() + .name("big-steward".to_string()) + .spawn(move || steward::run(&state)) + .expect("one steward thread") + }; + let mut workers = Vec::with_capacity(self.state.config.workers); for _ in 0..self.state.config.workers { let rx = Arc::clone(&rx); @@ -397,12 +418,17 @@ impl Server

{ // peer it has given up waiting for. Err(e) if transient(&e) => continue, Err(e) => { + // A listener that has failed is a server that is stopping, whatever the + // caller asked for; the steward has to hear that too, or it outlives the + // port it was serving. + self.state.stopping.store(true, Ordering::Relaxed); drop(tx); drop(shed_tx); for w in workers { let _ = w.join(); } let _ = shedder.join(); + let _ = steward.join(); return Err(e); } } @@ -414,6 +440,7 @@ impl Server

{ let _ = w.join(); } let _ = shedder.join(); + let _ = steward.join(); Ok(()) } diff --git a/crates/big-http/src/metrics.rs b/crates/big-http/src/metrics.rs index 055ac1b..5d6281d 100644 --- a/crates/big-http/src/metrics.rs +++ b/crates/big-http/src/metrics.rs @@ -62,6 +62,11 @@ pub struct ServerMetrics { /// clients that got a connection reset instead of a `503`, which is a different support /// call from the one a `503` produces. connections_shed_unspoken: AtomicU64, + /// Pages the steward has given back to the filesystem, and the passes on which it could + /// not because a query was in flight. The pair is the whole story: a node whose file never + /// shrinks and whose blocked count climbs is a node that is never idle for long enough. + pages_reclaimed: AtomicU64, + reclaim_blocked: AtomicU64, /// Handshakes that never completed: a wrong CA, an expired certificate, a client that hung /// up, or plaintext sent to a TLS port. tls_handshakes_failed: AtomicU64, @@ -92,6 +97,17 @@ impl ServerMetrics { self.connections_rejected.fetch_add(1, Ordering::Relaxed); } + /// Trailing free pages handed back to the filesystem. + pub fn pages_reclaimed(&self, pages: u64) { + self.pages_reclaimed.fetch_add(pages, Ordering::Relaxed); + } + + /// A pass that found something to give back and could not, because a read transaction was + /// open. An ordinary outcome on a busy node, and only worth looking at as a rate. + pub fn reclaim_blocked(&self) { + self.reclaim_blocked.fetch_add(1, Ordering::Relaxed); + } + /// A connection closed without being told it was being shed, because the thread that says so /// was itself saturated. pub fn connection_shed_unspoken(&self) { @@ -166,6 +182,21 @@ impl ServerMetrics { "Connections closed without a 503, because the thread that answers one was busy.", g(&self.connections_shed_unspoken), ); + counter( + &mut out, + "big_pages_reclaimed_total", + "Trailing free pages given back to the filesystem while serving. Zero on a node \ + started without --reclaim, where only an offline `big compact` shrinks the file.", + g(&self.pages_reclaimed), + ); + counter( + &mut out, + "big_reclaim_blocked_total", + "Passes that found pages to give back and could not, because a query was in \ + flight. Climbing while big_pages_reclaimed_total stays flat is a node that is \ + never idle long enough to shrink.", + g(&self.reclaim_blocked), + ); counter( &mut out, "big_http_tls_handshakes_failed_total", @@ -447,7 +478,7 @@ pub fn render_keys(out: &mut String, k: &big_embed::KeyStats) { /// **`big_cluster_copies_behind` is the number to alert on.** It is redundancy this cluster has /// lost and will not get back until somebody runs `POST /repair`; everything else here recovers /// on its own. -pub fn render_cluster(out: &mut String, c: &big_cluster::counters::Snapshot) { +pub fn render_cluster(out: &mut String, c: &big_cluster::counters::Snapshot, balancing: bool) { gauge( out, "big_cluster_nodes", @@ -460,6 +491,81 @@ pub fn render_cluster(out: &mut String, c: &big_cluster::counters::Snapshot) { "Copies the agreement has marked behind. Redundancy lost until a repair is run.", c.behind as u64, ); + // **The second number to alert on.** A move takes seconds; a range that has been moving + // for minutes is one the balancer will plan nothing around, cluster-wide, until an + // operator runs `POST /admin/cluster/cancel`. Nothing else here says so. + gauge( + out, + "big_cluster_ranges_moving", + "Ranges with a move in flight. Nonzero for longer than a move takes means the balancer \ + is silenced until the move is cancelled or finished.", + c.moving as u64, + ); + gauge( + out, + "big_cluster_schema_ready", + "1 when the schema leader holds the row-key namespace. 0 is a handover in progress, \ + during which no new row key can be assigned.", + c.schema_ready as u64, + ); + gauge( + out, + "big_cluster_schema_handover_blocked", + "1 when handing the namespace to an elected successor found two surviving nodes \ + disagreeing about a row id. Nobody assigns row ids until an operator looks.", + c.handover_blocked as u64, + ); + gauge( + out, + "big_cluster_balancer_enabled", + "1 when this node was started with --balance and may reshape the cluster by itself.", + balancing as u64, + ); + counter( + out, + "big_cluster_peer_load_unanswered_total", + "Times a peer did not say what it weighs. To the balancer that node is neither a \ + source nor a destination, so a climbing count is a cluster that will not reshape.", + c.counts.load_unanswered, + ); + counter( + out, + "big_cluster_balance_moves_total", + "Ranges the balancer moved.", + c.counts.balance_moves, + ); + counter( + out, + "big_cluster_balance_splits_total", + "Tails the balancer cut for a node with nothing.", + c.counts.balance_splits, + ); + counter( + out, + "big_cluster_balance_admits_total", + "Learners the balancer made full members once they answered and had caught up.", + c.counts.balance_admits, + ); + counter( + out, + "big_cluster_balance_errors_total", + "Balancing steps that failed.", + c.counts.balance_errors, + ); + counter( + out, + "big_cluster_balance_drops_failed_total", + "Moves that landed but whose source could not let go of its copy. Space that is not \ + reclaimed until somebody looks.", + c.counts.balance_drops_failed, + ); + counter( + out, + "big_cluster_move_cancel_failed_total", + "Moves that failed and could not be called off either. Each one is a range still \ + marked moving.", + c.counts.move_cancel_failed, + ); gauge( out, "big_cluster_serving", diff --git a/crates/big-http/src/routes/mod.rs b/crates/big-http/src/routes/mod.rs index c784073..303427c 100644 --- a/crates/big-http/src/routes/mod.rs +++ b/crates/big-http/src/routes/mod.rs @@ -235,6 +235,8 @@ enum Target<'a> { PeerDelete, PeerIntern, PeerAllocate, + PeerSchemaStepDown, + PeerReserve, PeerNextRecord, PeerDdl, PeerDigest, @@ -297,6 +299,8 @@ impl<'a> Target<'a> { | Self::PeerDelete | Self::PeerIntern | Self::PeerAllocate + | Self::PeerSchemaStepDown + | Self::PeerReserve | Self::PeerNextRecord | Self::PeerDdl | Self::PeerRaft @@ -395,6 +399,8 @@ impl<'a> Target<'a> { | Self::PeerDelete | Self::PeerIntern | Self::PeerAllocate + | Self::PeerSchemaStepDown + | Self::PeerReserve | Self::PeerDdl | Self::PeerRaft | Self::PeerFragmentPut @@ -432,6 +438,8 @@ fn resolve<'a>(method: &str, segments: &[&'a str]) -> Option> { ("POST", ["internal", "delete"]) => Target::PeerDelete, ("POST", ["internal", "intern"]) => Target::PeerIntern, ("POST", ["internal", "allocate"]) => Target::PeerAllocate, + ("POST", ["internal", "schema", "step-down"]) => Target::PeerSchemaStepDown, + ("POST", ["internal", "reserve"]) => Target::PeerReserve, ("POST", ["internal", "next-record"]) => Target::PeerNextRecord, ("POST", ["internal", "ddl"]) => Target::PeerDdl, ("POST", ["internal", "digest"]) => Target::PeerDigest, @@ -541,7 +549,7 @@ pub fn dispatch(ctx: &Ctx<'_, P>, req: &Request) -> Answered Target::Metrics => { let mut text = ctx.metrics.render(&ctx.api().metrics()); crate::metrics::render_keys(&mut text, &ctx.api().key_stats()); - crate::metrics::render_cluster(&mut text, &ctx.cluster.counters()); + crate::metrics::render_cluster(&mut text, &ctx.cluster.counters(), ctx.balance.enabled); let (verifications, hits, throttled) = ctx.auth.counters(); crate::metrics::render_auth(&mut text, verifications, hits, throttled); Response::text( @@ -570,6 +578,8 @@ pub fn dispatch(ctx: &Ctx<'_, P>, req: &Request) -> Answered Target::PeerDelete => peer_delete(ctx, req), Target::PeerIntern => peer_intern(ctx, req), Target::PeerAllocate => peer_allocate(ctx, req), + Target::PeerSchemaStepDown => peer_schema_step_down(ctx, req), + Target::PeerReserve => peer_reserve(ctx, req), Target::PeerNextRecord => peer_next_record(ctx, req), Target::PeerDdl => peer_ddl(ctx, req), Target::PeerDigest => peer_digest(ctx, req), diff --git a/crates/big-http/src/routes/peer.rs b/crates/big-http/src/routes/peer.rs index 196f00b..9abf20c 100644 --- a/crates/big-http/src/routes/peer.rs +++ b/crates/big-http/src/routes/peer.rs @@ -125,15 +125,20 @@ pub(super) fn peer_delete(ctx: &Ctx<'_, P>, req: &Request) - /// The schema leader's one job. /// -/// Answered by whichever node receives it, which is the leader because a coordinator sends it -/// nowhere else. There is no check here that this node *is* the leader: the config is what -/// decides that, every node reads the same file, and a check would turn a misconfiguration -/// that startup already refuses into a runtime error somewhere less useful. +/// **Checked, not assumed.** It used to be assumed: the leader was a name in the config, every +/// node read the same file, and a request could only have been sent here on purpose. The +/// leader is a field in the map now, and a coordinator holding a map one decision old sends +/// this to whoever led *then* - which is how two nodes come to intern at once, the one failure +/// that hands one string two row ids and that nothing downstream can see. So the request says +/// who it thinks leads, and this node says whether it agrees before it assigns anything. pub(super) fn peer_intern(ctx: &Ctx<'_, P>, req: &Request) -> Response { let request = match wire::InternRequest::decode(&req.body) { Ok(r) => r, Err(e) => return unreadable(&e), }; + if let Err(e) = ctx.cluster.check_schema_lead(request.led.as_ref()) { + return super::from_cluster(&e); + } let keys: Vec<&str> = request.keys.iter().map(String::as_str).collect(); match ctx.api().intern_keys(&request.table, &request.field, &keys) { Ok(rows) => { @@ -154,12 +159,52 @@ pub(super) fn peer_allocate(ctx: &Ctx<'_, P>, req: &Request) Ok(r) => r, Err(e) => return unreadable(&e), }; + if let Err(e) = ctx.cluster.check_schema_lead(request.led.as_ref()) { + return super::from_cluster(&e); + } match ctx.cluster.allocate_here(&request.table, request.count) { Ok(from) => Response::binary(wire::put_u64_body(from)), Err(e) => super::from_cluster(&e), } } +/// `POST /internal/reserve`: raise the agreement's ceiling on record ids for a table. +/// +/// Asked of the agreement's leader by a schema leader that is not it, once per block of ids. +/// The body is the same shape floors travel in - `(table, one past the highest)` - and the +/// answer is the epoch the ceiling landed at. +pub(super) fn peer_reserve(ctx: &Ctx<'_, P>, req: &Request) -> Response { + let floors = match wire::get_floors(&req.body) { + Ok(f) => f, + Err(e) => return unreadable(&e), + }; + let mut epoch = 0; + for (table, upto) in &floors { + match ctx.cluster.reserve_here(table, *upto) { + Ok(at) => epoch = at, + Err(e) => return super::from_cluster(&e), + } + } + Response::binary(wire::put_u64_body(epoch)) +} + +/// `POST /internal/schema/step-down`: stop interning, the namespace is being handed over. +/// +/// The body is the map epoch the move read. This node refuses to intern or allocate until the +/// map is past it - which, once the move lands, it is, and by then the map names somebody +/// else. Answered with the epoch, so the caller knows which one was heard. +pub(super) fn peer_schema_step_down( + ctx: &Ctx<'_, P>, + req: &Request, +) -> Response { + let epoch = match wire::get_u64_body(&req.body) { + Ok(e) => e, + Err(e) => return unreadable(&e), + }; + ctx.cluster.stand_down_schema(epoch); + Response::binary(wire::put_u64_body(epoch)) +} + /// One past the highest record id this node holds, which is its share of the answer the leader /// allocates above. pub(super) fn peer_next_record(ctx: &Ctx<'_, P>, req: &Request) -> Response { @@ -387,7 +432,7 @@ pub(super) fn cluster_rebalance(ctx: &Ctx<'_, P>, req: &Requ } match ctx.cluster.rebalance(&policy) { Ok(None) => Response::ok("{\"did\":null}".to_string()), - Ok(Some(what)) => Response::ok(format!("{{\"did\":{}}}", json::string(&what))), + Ok(Some(what)) => Response::ok(format!("{{\"did\":{}}}", json::string(&what.describe()))), Err(e) => from_cluster(&e), } } diff --git a/crates/big-http/src/steward.rs b/crates/big-http/src/steward.rs new file mode 100644 index 0000000..6e5b3b3 --- /dev/null +++ b/crates/big-http/src/steward.rs @@ -0,0 +1,340 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The thread that does a node's slow, optional work on its own behalf. +//! +//! **Not the agreement's thread.** `Controller::run` ticks every fifty milliseconds with the +//! raft lock held, and everything it does must be over before the next heartbeat is due. A +//! balancing step is the opposite kind of work: it copies a range over the network, proposes to +//! the agreement and waits for the proposal to land - seconds, with the raft lock taken and +//! released along the way. Run from inside the driver it would deadlock against itself; run +//! from a worker it would hold a request's thread for the life of a move. So it has a thread of +//! its own, owned by the server, and the server stops it the way it stops its workers. +//! +//! **Every job here is off until something turns it on.** A node that reshapes its cluster, or +//! its file, without being asked is a node whose behaviour an operator cannot predict. The +//! jobs read their switches from the configuration on every pass, do nothing while the switch +//! is off, and cost one clock read a second for the privilege. + +use crate::State; +use big_cluster::raft::{MoveState, RangeId}; +use big_pager::PagerMut; +use big_wire::log; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +/// How often the jobs are considered. +/// +/// A second, not the agreement's fifty milliseconds: a pass that gets past its gates asks every +/// member what it weighs, and a pass that acts costs seconds. Nothing here is urgent. +const PERIOD: Duration = Duration::from_secs(1); + +/// How often the sleep looks up to see whether the server is standing down. +/// +/// A stop is a stop: a thread that sleeps a whole period before noticing is a shutdown that +/// takes a second longer than it should, and a test that waits on one. +const WAKE: Duration = Duration::from_millis(50); + +/// After a job fails, how long before it is tried again - and the ceiling on that. +/// +/// A cluster that cannot balance because a peer is down is not helped by being asked again +/// every second, and the peer is not helped by a request a second from every leader it has had. +const BACKOFF_FROM: Duration = Duration::from_secs(1); +const BACKOFF_TO: Duration = Duration::from_secs(60); + +/// How long a range may sit in one state of a move before somebody is told. +/// +/// A move takes seconds. Ten minutes in the same state is not a slow move; it is a move whose +/// driver died, and a range marked moving silences the balancer cluster-wide until an operator +/// runs `POST /admin/cluster/cancel`. Said once per transition rather than every pass, and +/// never acted on: a canceller racing a slow-but-progressing move would throw away real work. +const STUCK_AFTER: Duration = Duration::from_secs(600); + +/// How much of the file has to be reclaimable before the tail is worth trying to give back. +/// +/// A quarter. Below that the holes are what copy-on-write always leaves and what the freelist +/// is there to reuse, and taking the write lock to hand back a few pages would be paying for +/// nothing. +const RECLAIM_AT_PERCENT: u64 = 25; + +/// And a floor, so that a database of a few hundred pages is left alone whatever the ratio: a +/// quarter of nothing is nothing, and an empty file has no tail worth trimming. +const RECLAIM_FLOOR_PAGES: u64 = 4_096; + +/// Runs until the server stands down. +pub(crate) fn run(state: &State

) { + let mut backoff = Backoff::default(); + let mut watch = Watch::default(); + while !state.stopping.load(Ordering::Relaxed) { + if let Some((range, in_state, for_)) = watch.observe(moving(state), Instant::now()) { + log::emit( + log::Level::Warn, + "move_stuck", + &[ + ("range", log::F::N(range)), + ("state", log::F::S(&format!("{in_state:?}"))), + ("for_secs", log::F::N(for_.as_secs())), + ], + ); + } + + // The handover first: while it is open nobody assigns row ids, and nothing the + // balancer could do is worth more than closing it. + let handed = match handover(state) { + Ok(None) => Ok(()), + Ok(Some(to)) => { + log::emit(log::Level::Info, "schema_handover", &[("to", log::F::S(&to))]); + Ok(()) + } + Err(e) => { + // A survivor that did not answer is retried; a survivor that contradicts + // another is not something a retry can fix, and it is said at the level it + // deserves. Both back off: the cluster is not helped by being asked again + // every second. + let level = if e.is_unreachable() { log::Level::Warn } else { log::Level::Error }; + log::emit(level, "schema_handover_failed", &[("error", log::F::S(&e.to_string()))]); + Err(()) + } + }; + + // Local, and gated on nothing but its own switch: it asks no peer and proposes + // nothing, so it runs on every node - clustered or not, leader or not. + reclaim(state); + + let wait = match balance(state) { + Ok(None) if handed.is_ok() => PERIOD, + Ok(None) => backoff.next(), + Ok(Some(did)) => { + log::emit(log::Level::Info, "balance_step", &[("did", log::F::S(&did.describe()))]); + PERIOD + } + Err(e) => { + let wait = backoff.next(); + log::emit( + log::Level::Warn, + "balance_error", + &[ + ("error", log::F::S(&e.to_string())), + ("retry_in_ms", log::F::N(wait.as_millis() as u64)), + ], + ); + wait + } + }; + if wait <= PERIOD { + backoff.reset(); + } + sleep_unless_stopping(state, wait); + } +} + +/// One balancing step, if this node is the one that decides and the policy allows it. +/// +/// **Only the agreement's leader**, and only when everything it has proposed has landed - the +/// same two gates `promotion` stands behind, for the same reason: a decision about a map that +/// has not settled is a decision about a state that is already gone. A node with no agreement +/// has nothing to balance. +fn balance( + state: &State

, +) -> big_cluster::Result> { + let policy = &state.config.balance; + if !policy.enabled { + return Ok(None); + } + let Some(controller) = state.cluster.controller() else { + return Ok(None); + }; + if !controller.is_leader() || !controller.settled() { + return Ok(None); + } + state.cluster.rebalance(policy) +} + +/// Finishes a handover of the row-key namespace the agreement decided, if one is open. +/// +/// **The agreement's leader, and only it.** It decided the successor, it is the one node that +/// can mark the successor ready, and it reaches every survivor. The successor itself does +/// nothing here: it is given the keys, and told when it holds them all. +fn handover( + state: &State

, +) -> big_cluster::Result> { + let Some(controller) = state.cluster.controller() else { + return Ok(None); + }; + if !controller.is_leader() || !controller.settled() { + return Ok(None); + } + state.cluster.finish_schema_handover() +} + +/// Gives trailing free pages back to the filesystem, when there are enough to be worth it. +/// +/// **The only thing that makes a served database smaller.** Copy-on-write leaves holes, the +/// freelist reuses them lowest-first so the tail drains, and this hands the drained tail back. +/// `big compact` does the same job offline and wants the exclusive lock, which means stopping +/// the node. +/// +/// Three things keep it out of the way. It does nothing until a quarter of the file is +/// reclaimable, so an ordinary database is never touched. It does nothing while a backup is +/// walking the file, because a backup holds a reader for its whole run: the horizon cannot +/// move, so there would be nothing to release and the attempt would only take the write lock. +/// And `ReadersActive` - a read transaction open anywhere in this process - is an ordinary +/// outcome rather than a failure, counted and retried on the next pass rather than logged. +fn reclaim(state: &State

) { + if !state.config.reclaim { + return; + } + if state.backing_up.load(Ordering::Relaxed) { + return; + } + let m = state.cluster.local().metrics(); + if m.page_count < RECLAIM_FLOOR_PAGES + || m.free_pages_reusable * 100 < m.page_count * RECLAIM_AT_PERCENT + { + return; + } + match state.cluster.local().reclaim() { + Ok(0) => {} + Ok(pages) => { + state.metrics.pages_reclaimed(pages); + log::emit(log::Level::Info, "reclaimed", &[("pages", log::F::N(pages))]); + } + // Not a failure: it means a query was in flight, which on a busy node is most of the + // time. The counter is what says whether "most" has become "always". + Err(_) => state.metrics.reclaim_blocked(), + } +} + +/// The one move in flight, if there is one. The map allows at most one. +fn moving(state: &State

) -> Option<(RangeId, MoveState)> { + state.cluster.map().ranges.iter().find_map(|r| r.moving.as_ref().map(|m| (r.id, m.state))) +} + +/// Sleeps for `wait`, waking early if the server stands down meanwhile. +fn sleep_unless_stopping(state: &State

, wait: Duration) { + let mut left = wait; + while !left.is_zero() && !state.stopping.load(Ordering::Relaxed) { + let step = left.min(WAKE); + std::thread::sleep(step); + left -= step; + } +} + +/// Doubling, capped, reset on success. Local to the thread: nothing about a retry belongs in +/// the map. +#[derive(Default)] +struct Backoff { + current: Option, +} + +impl Backoff { + fn next(&mut self) -> Duration { + let next = self.current.map_or(BACKOFF_FROM, |c| (c * 2).min(BACKOFF_TO)); + self.current = Some(next); + next + } + + fn reset(&mut self) { + self.current = None; + } +} + +/// Watches the one move in flight and says when it has been in one state for too long. +/// +/// **Local, and deliberately so.** The obvious place for "since when" is the map, and it would +/// be wrong there twice: the balancer's determinism rests on reading no clock, and a timestamp +/// in a replicated value is a clock every node reads differently. So the steward remembers +/// what it saw and when, and forgets both the moment the move changes state or ends. +#[derive(Default)] +struct Watch { + seen: Option<(RangeId, MoveState, Instant)>, + warned: bool, +} + +impl Watch { + /// Feeds one observation. `Some` exactly once per stuck state: the moment it has lasted + /// past the threshold, and not again until the move changes. + fn observe( + &mut self, + now_moving: Option<(RangeId, MoveState)>, + now: Instant, + ) -> Option<(RangeId, MoveState, Duration)> { + match (now_moving, self.seen) { + (None, _) => { + self.seen = None; + self.warned = false; + None + } + (Some((id, st)), Some((seen_id, seen_st, since))) if seen_id == id && seen_st == st => { + let for_ = now.saturating_duration_since(since); + if for_ >= STUCK_AFTER && !self.warned { + self.warned = true; + Some((id, st, for_)) + } else { + None + } + } + (Some((id, st)), _) => { + self.seen = Some((id, st, now)); + self.warned = false; + None + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backoff_doubles_to_a_ceiling_and_resets() { + let mut b = Backoff::default(); + assert_eq!(b.next(), Duration::from_secs(1)); + assert_eq!(b.next(), Duration::from_secs(2)); + assert_eq!(b.next(), Duration::from_secs(4)); + for _ in 0..10 { + b.next(); + } + assert_eq!(b.next(), BACKOFF_TO, "capped, however many times it fails"); + b.reset(); + assert_eq!(b.next(), Duration::from_secs(1), "a success starts the clock again"); + } + + /// A move that progresses is never reported; one that does not is reported once, and + /// again only if it stalls afresh in another state. + #[test] + fn a_stalled_move_is_reported_once_per_state() { + let mut w = Watch::default(); + let t0 = Instant::now(); + let seeding = Some((7, MoveState::Seeding)); + let cutover = Some((7, MoveState::Cutover)); + + assert_eq!(w.observe(seeding, t0), None, "first sight starts the clock"); + assert_eq!(w.observe(seeding, t0 + STUCK_AFTER / 2), None, "half way is not stuck"); + let stuck = w.observe(seeding, t0 + STUCK_AFTER); + assert_eq!(stuck.map(|(id, st, _)| (id, st)), Some((7, MoveState::Seeding))); + assert_eq!(w.observe(seeding, t0 + STUCK_AFTER * 2), None, "said once, not every pass"); + + // It moved on. The clock restarts with the new state, and warns again only if that + // one stalls too. + assert_eq!(w.observe(cutover, t0 + STUCK_AFTER * 2), None); + assert_eq!(w.observe(cutover, t0 + STUCK_AFTER * 2 + STUCK_AFTER / 2), None); + assert!(w.observe(cutover, t0 + STUCK_AFTER * 3).is_some()); + + // Finished: nothing to watch, nothing remembered. + assert_eq!(w.observe(None, t0 + STUCK_AFTER * 4), None); + assert_eq!(w.observe(seeding, t0 + STUCK_AFTER * 4), None, "a new move starts fresh"); + } +} diff --git a/crates/big-http/tests/cluster.rs b/crates/big-http/tests/cluster.rs index a4162b6..3f5790f 100644 --- a/crates/big-http/tests/cluster.rs +++ b/crates/big-http/tests/cluster.rs @@ -87,6 +87,14 @@ fn a_replicated_group() -> (SocketAddr, SocketAddr, SocketAddr) { [[node]]\nname = \"a-third\"\naddr = \"{third}\"\nreplica = \"a\"\n" ); start(&file, &[("a", a), ("a-spare", spare), ("a-third", third)]); + // **Wait for the agreement before asking anything of it.** Every node here holds a + // replicated range, so every one of them is fenced: until it has heard from the agreement + // it cannot know it has not already been replaced, and it refuses rather than risk being + // the second node answering for one range. On the shipped clocks that is an election, and + // an election is what a replicated cluster needs before it can serve at all. + waiting("an elected leader", std::time::Duration::from_secs(15), || { + ready(a).contains(r#""leader":""#) + }); (a, spare, third) } @@ -757,6 +765,9 @@ fn a_write_that_misses_a_copy_stands_and_says_so() { // `a-third` is configured and never started, so a majority is still two of three and the // agreement works - it is one copy that is missing, not the quorum. start(&file, &[("a", a), ("a-spare", spare)]); + waiting("an elected leader", std::time::Duration::from_secs(15), || { + ready(a).contains(r#""leader":""#) + }); // A schema change reaches the copies it can and names the one it cannot. let (status, body) = send(a, "POST", "/table/tx", ""); @@ -859,6 +870,7 @@ fn brisk() -> (Timing, Leases) { Leases { serve_for: std::time::Duration::from_millis(300), promote_after: std::time::Duration::from_millis(900), + ..Leases::default() }, ) } @@ -876,6 +888,17 @@ fn a_group_of_three() -> (String, Vec) { (file, ports) } +/// The same as `brisk`, and the agreement is also allowed to move the row-key namespace. +/// +/// Fifteen seconds in a deployment, a quarter of a second here, and for the same reason the +/// other clocks are shortened: what these tests are about is the rule, and the rule does not +/// depend on the number. The ordering does, and it is kept - the namespace still waits longer +/// than a range does. +fn brisk_electing() -> (Timing, Leases) { + let (timing, leases) = brisk(); + (timing, Leases { move_schema_after: Some(leases.promote_after * 3), ..leases }) +} + /// A node a test can take away. /// /// `close` drops the listener rather than only stopping the loop: a port that accepts and never @@ -930,6 +953,32 @@ fn start_with( timing: Timing, leases: Leases, state: Option, +) -> Node { + start_configured(file, name, addr, timing, leases, state, ServerConfig::default()) +} + +/// On the brisk clocks, with the balancer switched on - the one thing the steward acts on. +fn start_balancing(file: &str, name: &str, addr: SocketAddr) -> Node { + let (timing, leases) = brisk(); + let config = ServerConfig { + balance: big_cluster::balance::Policy { + enabled: true, + ..big_cluster::balance::Policy::default() + }, + ..ServerConfig::default() + }; + start_configured(file, name, addr, timing, leases, None, config) +} + +/// The same, with the server's configuration spelled out as well. +fn start_configured( + file: &str, + name: &str, + addr: SocketAddr, + timing: Timing, + leases: Leases, + state: Option, + server: ServerConfig, ) -> Node { let config = ClusterFile::parse(file).unwrap().for_node(Some(name), "").unwrap(); let store: Box = match state { @@ -939,8 +988,7 @@ fn start_with( let cluster = Cluster::with_timing(Api::in_memory().unwrap(), config, None, store, timing, leases) .unwrap(); - let server = - Server::bind_cluster(cluster, addr, ServerConfig::default()).expect("the port was free"); + let server = Server::bind_cluster(cluster, addr, server).expect("the port was free"); let running = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); let flag = std::sync::Arc::clone(&running); let done = std::thread::spawn(move || { @@ -1009,6 +1057,49 @@ fn a_range_fails_over_when_its_primary_dies() { }); } +/// **A range still fails over after the leader has compacted its log.** +/// +/// The guard in front of a promotion compared the commit index with the length of the vector +/// holding the log's suffix, and the two could never agree again once anything had been +/// compacted away - so every promotion after the first compaction was refused, silently, on a +/// cluster reporting itself healthy. The rule is proven against a simulated agreement in +/// `big-cluster/tests/raft.rs`; this is the one test that runs it inside the real driver +/// thread, against a log the real leader compacted on its own. +#[test] +fn a_range_fails_over_after_the_log_has_been_compacted() { + let (file, ports) = a_group_of_three(); + // **The copies elect a leader before the primary arrives.** Only the leader compacts its + // log - a follower's base moves only when it is handed a snapshot - so the node that dies + // must not be the one that decides, or its successor would decide from an uncompacted log + // and this test would pass with the bug in place. `a` is the primary by the file; the + // agreement's leader has to be one of the other two. + let _b = start_agreeing(&file, "b", ports[1]); + let _c = start_agreeing(&file, "c", ports[2]); + until("a leader among the copies", || ready(ports[1]).contains(r#""leader":""#)); + let mut a = start_agreeing(&file, "a", ports[0]); + until("the primary to join", || ready(ports[0]).contains(r#""leader":""#)); + let leader = leader_of(&ports); + assert_ne!(leader, ports[0], "the primary joined a settled agreement; it must not lead it"); + + ok(ports[0], "POST", "/table/tx", ""); + ok(ports[0], "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(ports[0], "POST", "/table/tx/import", "amount 1 5\namount 2 5\n"); + + // Enough committed decisions that the leader compacts. Every split and every merge is one + // entry, and with every voter up the leader drops whatever is more than `KEEP_ENTRIES` + // behind - so more than that many, with room, is a log that has certainly been cut. + for _ in 0..(big_cluster::raft::Raft::KEEP_ENTRIES / 2 + 8) { + ok(leader, "POST", "/admin/cluster/split?at=900", ""); + ok(leader, "POST", "/admin/cluster/merge?range=0", ""); + } + + a.close(); + until("the range to move after a compaction", || { + let (status, body) = send(ports[1], "POST", "/table/tx/query", "Count(All())"); + status == 200 && body == r#"{"count":2}"# + }); +} + /// **A node cut off from the agreement stops answering for its range.** /// /// This is the half that makes the other half safe. A failure detector alone cannot tell a @@ -1193,6 +1284,8 @@ fn a_second_batch_of_keys_does_not_collide_with_the_first() { let mut a_spare = start_agreeing(&file, "a-spare", spare); start_agreeing(&file, "a", a).forget(); start_agreeing(&file, "b", b).forget(); + // `a` holds a replicated range, so it is fenced until it has heard from the agreement. + until("an elected leader", || ready(a).contains(r#""leader":""#)); ok(a, "POST", "/table/tx", ""); ok(a, "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); @@ -1371,7 +1464,7 @@ fn a_node_that_restarts_keeps_what_it_agreed() { // The file was read rather than started fresh: the state on disk says so. let written = std::fs::read(dir.path().join("c.raft")).unwrap(); - assert!(written.starts_with(b"BIGRAFT2"), "the agreement wrote no state for `c`"); + assert!(written.starts_with(b"BIGRAFT3"), "the agreement wrote no state for `c`"); // Reads never stopped: the copy serving the range was never the one taken away. assert_eq!(ok(ports[0], "POST", "/table/tx/query", "Count(All())"), r#"{"count":1}"#); @@ -1415,6 +1508,7 @@ fn a_serving_node_that_comes_back_empty_is_only_found_by_verify() { let leases = Leases { serve_for: std::time::Duration::from_secs(5), promote_after: std::time::Duration::from_secs(30), + ..Leases::default() }; let mut a = start_with(&file, "a", ports[0], timing, leases, None); let _b = start_with(&file, "b", ports[1], timing, leases, None); @@ -1726,6 +1820,37 @@ fn a_node_joins_as_a_learner_and_is_promoted_when_it_has_something_to_serve() { }); } +/// **A node that joined is admitted by the steward**, once it is answering for data and holds +/// the log - and by nothing else. The test above promotes by hand; this one starts the fourth +/// node for real, with a file that names the whole cluster and itself, and waits for the +/// agreement's leader to decide it counts. +#[test] +fn a_learner_that_has_caught_up_is_admitted_by_the_steward() { + let (a, b, spare, d) = (free_port(), free_port(), free_port(), free_port()); + // Named, so that a node whose file describes four nodes is recognised by three whose files + // describe three: without a `cluster_id` the shape of the file is what nodes check. + let file = format!("cluster_id = \"grows\"\n{}", three(a, b, spare)); + let _a = start_balancing(&file, "a", a); + let _b = start_balancing(&file, "b", b); + let _spare = start_balancing(&file, "a-spare", spare); + until("an elected leader", || ready(a).contains(r#""leader":""#)); + + let leader = leader_of(&[a, b, spare]); + ok(leader, "POST", &format!("/admin/cluster/node?name=d&addr={d}"), ""); + + // The node itself, seeded as a copy of `a` - the shape that parses without overlapping + // anybody's range, and one the agreement overwrites the moment it is heard from. + let grown = format!("{file}[[node]]\nname = \"d\"\naddr = \"{d}\"\nreplica = \"a\"\n"); + let _d = start_balancing(&grown, "d", d); + + until("the steward to admit it", || { + let seen = ok(a, "GET", "/cluster/topology", ""); + seen.contains(r#""name":"d","addr""#) && !seen.contains(r#""state":"learner""#) + }); + let metrics = ok(leader, "GET", "/metrics", ""); + assert!(metrics.contains("big_cluster_balance_admits_total 1"), "{metrics}"); +} + /// **A node that still holds a range is not removed**, because removing it removes the only /// copy of what it holds and leaves the map naming a node nobody talks to. #[test] @@ -1945,6 +2070,55 @@ fn a_cancelled_move_leaves_the_range_where_it_was() { assert!(after.contains(r#""shards":"64..","primary":"b""#), "{after}"); } +/// **Switched on, the balancer needs no operator.** The same step the test below asks for by +/// hand, taken by the steward on the agreement's leader - and only there - because the policy +/// says it may. Draining `b` is the only instruction anybody gives. +#[test] +fn the_balancer_runs_by_itself_when_switched_on() { + let (a, b, spare) = (free_port(), free_port(), free_port()); + let file = three(a, b, spare); + let _a = start_balancing(&file, "a", a); + let _b = start_balancing(&file, "b", b); + let _spare = start_balancing(&file, "a-spare", spare); + until("an elected leader", || ready(a).contains(r#""leader":""#)); + + ok(a, "POST", "/table/tx", ""); + ok(a, "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(a, "POST", "/table/tx/import", &format!("amount {} 5\n", 70 * (1 << 20))); + + let leader = leader_of(&[a, b, spare]); + ok(leader, "POST", "/admin/cluster/drain?name=b", ""); + until("the steward to move the range off the draining node", || { + !ok(a, "GET", "/cluster/topology", "").contains(r#""shards":"64..","primary":"b""#) + }); + // The records came with it. + assert_eq!(ok(a, "POST", "/table/tx/query", "Count(All())"), r#"{"count":1}"#); + + // **And it stops.** `b` holds nothing, so the next pass finds nothing to do, and a + // draining node is never a destination - which is exactly the state in which it can be + // removed for good. A removal refused is one the move has not finished landing. + until("the move to be over and `b` removable", || { + send(leader, "DELETE", "/admin/cluster/node?name=b", "").0 == 200 + }); +} + +/// The steward is stopped the way the workers are, and no slower. A node asked to stand down +/// comes back within a wake of the steward's sleep, not a whole period of it. +#[test] +fn a_node_stands_down_promptly_with_its_steward() { + let (file, ports) = a_group_of_three(); + let mut a = start_balancing(&file, "a", ports[0]); + until("the node to answer", || send(ports[0], "GET", "/ready", "").0 == 200); + + let started = std::time::Instant::now(); + a.close(); + assert!( + started.elapsed() < std::time::Duration::from_secs(1), + "stood down in {:?}", + started.elapsed() + ); +} + /// **The balancer, over real nodes.** It decides from what the nodes actually weigh, and one /// call does one thing - so a cluster that needs several steps takes several calls, each /// against facts gathered afresh. @@ -1985,11 +2159,43 @@ fn rebalancing_takes_a_range_off_a_draining_node_and_then_stops() { let again = ok(leader, "POST", "/admin/cluster/rebalance?force=true", ""); assert_eq!(again, r#"{"did":null}"#); + // The step was counted where an operator scrapes, and the move is over rather than + // lingering as a range marked moving - the state that silences every step after it. + let metrics = ok(leader, "GET", "/metrics", ""); + assert!(metrics.contains("big_cluster_balance_moves_total 1"), "{metrics}"); + assert!(metrics.contains("big_cluster_ranges_moving 0"), "{metrics}"); + assert!(metrics.contains("big_cluster_balance_drops_failed_total 0"), "{metrics}"); + // Which is exactly the state in which it can be removed for good. let (status, said) = send(leader, "DELETE", "/admin/cluster/node?name=b", ""); assert_eq!(status, 200, "{said}"); } +/// **A peer that does not say what it weighs is counted, not just skipped.** To the balancer +/// that node is neither a source nor a destination, so a cluster with one silent member is a +/// cluster that quietly stops reshaping - and the only way an operator tells that apart from a +/// cluster with nothing to do is this number. +#[test] +fn a_peer_that_does_not_answer_the_balancer_is_counted() { + let (a, b, spare) = (free_port(), free_port(), free_port()); + let file = three(a, b, spare); + let mut nodes = [ + start_agreeing(&file, "a", a), + start_agreeing(&file, "b", b), + start_agreeing(&file, "a-spare", spare), + ]; + until("an elected leader", || ready(a).contains(r#""leader":""#)); + let leader = leader_of(&[a, b, spare]); + + // Take away a node that is not deciding, so the one that is keeps deciding. + let silent = [a, b, spare].into_iter().position(|p| p != leader).expect("two others"); + nodes[silent].close(); + + ok(leader, "POST", "/admin/cluster/rebalance?force=true", ""); + let metrics = ok(leader, "GET", "/metrics", ""); + assert!(metrics.contains("big_cluster_peer_load_unanswered_total 1"), "{metrics}"); +} + /// Off unless asked. A cluster that reshapes itself unasked is one whose shape an operator /// cannot predict, so the policy ships disabled and `?force=true` is what an operator uses. #[test] @@ -2009,6 +2215,182 @@ fn the_balancer_does_nothing_until_it_is_switched_on() { assert_eq!(ok(leader, "POST", "/admin/cluster/rebalance", ""), r#"{"did":null}"#); } +/// **The agreement gives the row-key namespace away when its holder dies, and nothing is +/// handed the same meaning twice.** +/// +/// Four claims, and the middle two are what make the first one safe. The namespace moves. In +/// the gap before the successor holds every key, a write with a *new* key is refused rather +/// than given an id somebody else might also be giving out. Once it is ready, a key that +/// already had a row id has the *same* one - which is the proof that the keys came across from +/// the survivors rather than being invented again. And no record id is reused across the whole +/// thing. +#[test] +fn the_agreement_moves_the_namespace_when_its_holder_dies() { + let (a, b, spare) = (free_port(), free_port(), free_port()); + let file = three(a, b, spare); + let (timing, leases) = brisk_electing(); + let mut a_node = start_with(&file, "a", a, timing, leases, None); + let _b = start_with(&file, "b", b, timing, leases, None); + let _spare = start_with(&file, "a-spare", spare, timing, leases, None); + until("an elected leader", || ready(b).contains(r#""leader":""#)); + + ok(a, "POST", "/table/tx", ""); + ok(a, "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(a, "POST", "/table/tx/field/country?kind=set", ""); + ok(a, "POST", "/sql", "INSERT INTO tx (amount, country) VALUES (5, 'GB')"); + ok(a, "POST", "/sql", "INSERT INTO tx (amount, country) VALUES (7, 'FR')"); + + // What `GB` means before the handover. It has to mean the same afterwards: a successor + // that invented a second row id for it would answer a `GroupBy` with two groups that are + // one group, and nothing downstream could tell. + let before = ok(b, "POST", "/table/tx/query", "GroupBy(All(), field=\"country\")"); + assert!(before.contains("GB") && before.contains("FR"), "{before}"); + + // The schema leader dies. Nobody is told; nothing is edited. + a_node.close(); + + // 1. It moves, and to a node that is answering. + until("the namespace to move", || { + let seen = ok(b, "GET", "/cluster/topology", ""); + seen.contains(r#""schema_leader":"b""#) || seen.contains(r#""schema_leader":"a-spare""#) + }); + + // 2. Once the successor holds the keys, an existing key resolves to the id it always had. + until("the successor to finish taking it over", || { + send(b, "POST", "/sql", "INSERT INTO tx (amount, country) VALUES (9, 'GB')").0 == 200 + }); + let after = ok(b, "POST", "/table/tx/query", "GroupBy(All(), field=\"country\")"); + assert_eq!( + after.matches("GB").count(), + 1, + "one group for `GB`, not two - the keys came across: {after}" + ); + + // 3. A key nobody has ever seen works too, now that somebody holds the namespace. + ok(b, "POST", "/sql", "INSERT INTO tx (amount, country) VALUES (11, 'DE')"); + + // 4. No record id was handed out twice: four inserts, four records. + until("both survivors to agree on the count", || { + [b, spare] + .iter() + .all(|p| send(*p, "POST", "/table/tx/query", "Count(All())").1 == r#"{"count":4}"#) + }); + + // And the node that was deposed is marked behind: it may hold row ids it interned for + // writes that never landed anywhere, so it must be repaired before it is trusted again. + assert!(ready(b).contains(r#""behind":["a"]"#), "{}", ready(b)); +} + +/// **A record id promised by a leader that died is never promised again.** +/// +/// The floor a leader allocates from lives in its memory, and a successor - here the same node, +/// restarted with an empty database - would start from what is on disk and hand out ids the +/// old leader had already given to writes that may not have landed. So the leader commits a +/// ceiling through the agreement before it hands out anything under it, in blocks, and a +/// fresh leader starts at the ceiling. The tell is the id: it jumps to the block boundary +/// rather than continuing from the highest record anybody holds. +#[test] +fn a_restarted_schema_leader_starts_above_everything_it_may_have_promised() { + let dir = tempfile::tempdir().unwrap(); + let (file, ports) = a_group_of_three(); + let state = |name: &str| Some(dir.path().join(format!("{name}.raft"))); + let (timing, leases) = brisk(); + + let mut a = start_with(&file, "a", ports[0], timing, leases, state("a")); + let _b = start_with(&file, "b", ports[1], timing, leases, state("b")); + let _c = start_with(&file, "c", ports[2], timing, leases, state("c")); + until("an elected leader", || ready(ports[1]).contains(r#""leader":""#)); + + ok(ports[1], "POST", "/table/tx", ""); + ok(ports[1], "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(ports[1], "POST", "/sql", "INSERT INTO tx (amount) VALUES (5)"); + ok(ports[1], "POST", "/sql", "INSERT INTO tx (amount) VALUES (7)"); + assert_eq!( + ok(ports[1], "GET", "/table/tx/records?limit=5", ""), + r#"{"records":[0,1],"next":null}"#, + "two ids, from zero, as always" + ); + + // The schema leader goes away and comes back with its agreement state and nothing else - + // a replaced disk. Its floor was in memory; the ceiling it committed was not. + a.close(); + until("the copy that went away to be marked behind", || { + ready(ports[1]).contains(r#""behind":["a"]"#) + }); + let _a_again = start_with(&file, "a", ports[0], timing, leases, state("a")); + until("the restarted node to answer", || send(ports[0], "GET", "/ready", "").0 == 200); + // Caught up first: it is still a copy of the range every write goes to, and a copy with + // no table in it refuses the write. That is the repair's job, not this test's subject. + until("the restarted copy to be repaired", || { + send(ports[1], "POST", "/repair", "").1.contains(r#""outcome":"caught up""#) + }); + until("the leader to allocate again", || { + send(ports[1], "POST", "/sql", "INSERT INTO tx (amount) VALUES (9)").0 == 200 + }); + + // Not 2. The highest record anybody holds is 1, and a leader starting from there would be + // free to re-issue whatever the old one promised in between. + let ids = ok(ports[1], "GET", "/table/tx/records?after=1&limit=5", ""); + assert!(ids.contains("65536"), "the next id starts at the committed block: {ids}"); +} + +/// **A schema leader that loses the agreement stops assigning row ids**, even when the range +/// it holds is not fenced. A range with no copy is never fenced because nothing could take it; +/// the namespace can always be given to somebody else, so the node that holds it acts only +/// while it can prove it still does. What it costs is exactly one thing: a write with a key +/// nobody has seen waits. A write whose keys are known lands as it always did. +#[test] +fn the_schema_leader_stops_interning_when_it_loses_the_agreement() { + let (a, b, spare) = (free_port(), free_port(), free_port()); + // `a` leads the schema and holds a range of one copy; the copy that makes the agreement + // run at all is of `b`'s range. So when `b` and its copy go, `a` keeps serving its own + // range - and cannot know whether the namespace is still its. + let file = format!( + "schema_leader = \"a\"\n\ + [[node]]\nname = \"a\"\naddr = \"{a}\"\nshards = \"0..64\"\n\ + [[node]]\nname = \"b\"\naddr = \"{b}\"\nshards = \"64..\"\n\ + [[node]]\nname = \"b-spare\"\naddr = \"{spare}\"\nreplica = \"b\"\n" + ); + let _a = start_agreeing(&file, "a", a); + let mut b_node = start_agreeing(&file, "b", b); + let mut spare_node = start_agreeing(&file, "b-spare", spare); + until("an elected leader", || ready(a).contains(r#""leader":""#)); + + ok(a, "POST", "/table/tx", ""); + ok(a, "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(a, "POST", "/table/tx/field/country?kind=set", ""); + ok(a, "POST", "/table/tx/import", "amount 1 5\ncountry 1 GB\n"); + + b_node.close(); + spare_node.close(); + + // A key nobody has seen needs the leader, and `a` can no longer prove it is one. Whether + // `a` led the agreement or followed it makes no difference: a leader's lease is renewed + // by a majority it no longer has, a follower's by a leader it no longer hears. + // + // A fresh key every time. The lease outlives the closes by a moment, and a key interned + // in that moment is a key this node knows from then on - a second attempt with the same + // one would need no leader and prove nothing. + let mut attempt = 100u64; + until("the schema lease to run out", || { + attempt += 1; + let body = format!("amount {attempt} 7\ncountry {attempt} K{attempt}\n"); + let (status, body) = send(a, "POST", "/table/tx/import", &body); + status == 503 && body.contains(r#""code":"schema_lease_lost""#) + }); + + // Still serving its own range: that range has no copy, so nothing could have taken it, + // and the lease it lost is the namespace's and not the range's. + assert!(ready(a).contains(r#""serving":true"#), "{}", ready(a)); + + // A key it already knows needs nobody. The range is its own, and the write lands. (A + // count over the whole table would fan out to `b`'s range, which is the part of the + // cluster that is actually gone - so the import's own answer is the evidence.) + let (status, body) = send(a, "POST", "/table/tx/import", "amount 3 9\ncountry 3 GB\n"); + assert_eq!(status, 200, "{body}"); + assert!(body.contains(r#""imported":2"#), "{body}"); +} + /// **The row-key namespace moves, and nothing is handed the same id twice.** /// /// This is the one change that corrupts rather than fails. A successor that started from what @@ -2059,4 +2441,37 @@ fn the_schema_leader_moves_without_reissuing_anything() { ); let after = ok(b, "POST", "/table/tx/query", "GroupBy(All(), field=\"country\")"); assert!(after.contains("US"), "a key invented after the handover works too: {after}"); + + // **And the old leader refuses to intern**, rather than answering a coordinator that still + // holds the map from before the move. Two nodes interning at once is the one failure that + // hands one string two row ids, and this refusal is what keeps it to one: it used to be + // assumed rather than checked, because the leader was a name in a file. + let fingerprint = + ClusterFile::parse(&file).unwrap().for_node(Some("a"), "").unwrap().fingerprint(); + let stale = |leader: usize| { + big_cluster::wire::InternRequest { + table: "tx".to_string(), + field: "country".to_string(), + keys: vec!["DE".to_string()], + led: Some(big_cluster::wire::Led { epoch: 1, leader }), + } + .encode() + }; + // Sent to `a`, which by its own map no longer leads. + let (status, body) = send_bytes(a, "/internal/intern", &stale(0), fingerprint); + assert_eq!(status, 503, "{}", String::from_utf8_lossy(&body)); + assert!( + String::from_utf8_lossy(&body).contains("not_schema_leader"), + "{}", + String::from_utf8_lossy(&body) + ); + // Sent to `b`, which leads - by a sender that believes `a` does. Refused too: the answer + // names who leads so the sender can ask the right node on purpose rather than by luck. + let (status, body) = send_bytes(b, "/internal/intern", &stale(0), fingerprint); + assert_eq!(status, 503, "{}", String::from_utf8_lossy(&body)); + assert!( + String::from_utf8_lossy(&body).contains("not_schema_leader"), + "{}", + String::from_utf8_lossy(&body) + ); } diff --git a/crates/big-http/tests/operability.rs b/crates/big-http/tests/operability.rs index dcbd3b9..4947719 100644 --- a/crates/big-http/tests/operability.rs +++ b/crates/big-http/tests/operability.rs @@ -163,6 +163,11 @@ fn metrics_render_in_the_prometheus_text_format() { assert!(r.body.contains("big_page_count "), "{}", r.body); // The request that came before this one was counted. assert!(r.body.contains(r#"big_http_responses_total{class="2xx"} 1"#), "{}", r.body); + // The balancer's numbers are there even on a cluster of one, so a dashboard built against + // a single node keeps working when the second node arrives. Off, and nothing moving. + assert!(r.body.contains("big_cluster_balancer_enabled 0"), "{}", r.body); + assert!(r.body.contains("# TYPE big_cluster_ranges_moving gauge"), "{}", r.body); + assert!(r.body.contains("big_cluster_peer_load_unanswered_total 0"), "{}", r.body); // This server is in memory, and `MemPager` counts no I/O because it does none. Absent // rather than zero: a block of zeroes would read as a database nobody is writing to. @@ -208,6 +213,80 @@ fn metrics_report_what_the_storage_backend_did() { assert!(!r.body.contains("big_storage_read_bytes_total"), "{}", r.body); } +/// **A node started with `--reclaim` gives pages back while it is serving**, which nothing in +/// this tree could do before: `big compact` wants the exclusive lock, so shrinking a database +/// meant stopping the daemon. +/// +/// The whole chain, over a real socket: the steward notices the file has enough free space to +/// be worth it, `Api::reclaim` takes the write lock, the pager releases the runs that sit flush +/// against the end of the file, and the counters say what happened. +/// +/// **What this deliberately does not claim.** Only the *tail* is released - one live page near +/// the top pins every free page below it - so a database that is emptied and left alone stays +/// large, and the number here is small. Making the general case shrink means relocating the +/// fragments that sit above the free space, which is not built; the rule that makes the tail +/// drain at all is `the_lowest_reusable_page_is_allocated_first` in `big-pager`. +#[test] +#[cfg(unix)] +fn a_node_with_reclaim_on_gives_pages_back_while_serving() { + let dir = tempfile::tempdir().unwrap(); + let api = Api::open(dir.path().join("t.big")).unwrap(); + api.create_table("tx").unwrap(); + api.create_field("tx", "amount", big_db::catalog::FieldKind::Int, 32).unwrap(); + + // One record per shard rather than a pile in one: a bitmap holds a great many records in a + // handful of pages, and what fills a file is fragments. Enough of them to pass the floor + // the steward will not act below. + let records: Vec = (0..6_000u64).map(|i| i << 20).collect(); + let facts: Vec<_> = records + .iter() + .map(|r| big_embed::Fact::Int { field: "amount", record: *r, value: r % 1_000 }) + .collect(); + api.import("tx", &facts).unwrap(); + assert!(api.metrics().page_count > 4_096, "the fixture has to outgrow the reclaim floor"); + api.delete("tx", &records).unwrap(); + // One more commit. A transaction cannot reuse the pages it is itself freeing - they are + // not past the horizon until it has landed - so the delete's own bookkeeping went to the + // end of the file. The next write takes the lowest free page instead, which is what moves + // the live page off the tail and leaves something to give back. + api.import("tx", &[big_embed::Fact::Int { field: "amount", record: 1, value: 1 }]).unwrap(); + + let server = + Server::bind_with(api, "127.0.0.1:0", ServerConfig { reclaim: true, ..config() }).unwrap(); + let addr = server.local_addr().unwrap(); + let running = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let flag = std::sync::Arc::clone(&running); + let done = std::thread::spawn(move || { + let _ = server.serve_while(&flag); + }); + + // The steward runs on its own clock, so this is a wait rather than an assertion. + let started = std::time::Instant::now(); + loop { + let body = send(addr, "GET", "/metrics", "").body; + assert!(body.contains("# TYPE big_pages_reclaimed_total counter"), "{body}"); + if !body.contains("big_pages_reclaimed_total 0") { + break; + } + assert!( + started.elapsed() < std::time::Duration::from_secs(20), + "nothing was ever given back: {body}" + ); + std::thread::sleep(std::time::Duration::from_millis(50)); + } + + running.store(false, std::sync::atomic::Ordering::Relaxed); + let _ = done.join(); +} + +/// And off by default, which is the half that keeps every existing deployment as it was. +#[test] +fn a_node_without_reclaim_never_gives_a_page_back() { + let addr = spawn(1, config()); + let body = send(addr, "GET", "/metrics", "").body; + assert!(body.contains("big_pages_reclaimed_total 0"), "{body}"); +} + // --------------------------------------------------------------------------- // Credentials // --------------------------------------------------------------------------- diff --git a/crates/big-pager/readme.md b/crates/big-pager/readme.md index 70dbd37..0721008 100644 --- a/crates/big-pager/readme.md +++ b/crates/big-pager/readme.md @@ -445,6 +445,24 @@ in one or two passes. *Test:* `repeated_commits_do_not_grow_the_file_without_bound`. +**Pages a transaction gives back to itself feed this loop, and safely.** `WriteTxn::free` sends a +page the transaction itself allocated to a scratch list rather than the freelist, so `alloc` can +hand it straight back; `commit` calls `Freelist::absorb_scratch` on what is left, which lowers +`next_pgno` past pages flush with the end of the file and pushes the rest. **Without that, those +pages were lost outright:** materialised in the file, counted in `meta.page_count`, and named by +nothing — and a single one of them at the end of the file stops §7.4 releasing anything at all. +Every transaction whose tree shrinks produces some. + +They arrive stamped `freed_at = txn_id`, the committing transaction's own, which is above the +horizon by construction (all three horizon terms are `.min(base.txn_id)`). So they inflate +`entry_count` **without being allocatable**, and a shrinking commit may take one more chain page +from the tail than it otherwise would; the next commit, whose horizon has moved, takes it back. +The argument above is untouched — it rests on `alloc` taking from the front of a run, and +`absorb_scratch` only ever calls `push`. + +*Tests:* `a_transaction_that_abandons_its_own_pages_leaves_no_trace`, +`repeated_churning_commits_do_not_grow_the_file_without_bound`. + ### 7.4 Returning space to the filesystem `truncate_tail` drops reclaimable runs that sit flush against the end of the file, repeatedly, diff --git a/crates/big-pager/src/audit.rs b/crates/big-pager/src/audit.rs new file mode 100644 index 0000000..3556128 --- /dev/null +++ b/crates/big-pager/src/audit.rs @@ -0,0 +1,227 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Accounting for every page in the file: reachable, free, or neither. +//! +//! **Nothing here frees anything.** It answers one question - *how many pages does this file +//! contain that nothing points at and nothing has recorded as free* - and answers it as a +//! number, not as an action. A page like that is invisible to every other measurement: it is +//! counted in `page_count`, so the file is that much bigger, and it is in no freelist run, so +//! it is never handed out again. It is also enough, on its own, to stop +//! [`crate::Store::truncate_tail`] releasing anything at all - that walks down from the end of +//! the file and stops at the first page that is not a reusable free run, so one orphan at the +//! top pins every free page beneath it. +//! +//! **The complement is only as good as the mark.** A page class the mark phase forgets is a +//! page class this reports as leaked, so the enumeration in [`crate::Store::begin_audit`] is +//! the whole correctness argument, and the class that is easiest to forget is the one no other +//! walk in this tree visits: the trees an old *snapshot* still names. `Db::copy_to` skips them +//! deliberately, because it is building a new history; an audit is asserting what the existing +//! history still needs, so the one thing that walk drops is the one thing this must add. + +use crate::freelist::FreeRun; +use big_page::{Pgno, TxnId}; + +/// One bit per page. +/// +/// **A bitset rather than a count, and that is not an optimisation.** Under copy-on-write a +/// snapshot's tree shares almost every page with the current one, so the same page is +/// legitimately reached many times over across the root set - a counter would multiply +/// reachability and report a negative number of leaks. [`big_btree::visit_tree`] does not +/// deduplicate either: a page reached through two cells is yielded twice. Marking an +/// already-marked bit is a no-op, so neither matters here. +/// +/// [`big_btree::visit_tree`]: https://docs.rs/big-btree +#[derive(Clone, Debug)] +pub struct PageSet { + words: Vec, + len: u64, +} + +impl PageSet { + /// A set that can hold every page below `pages`, all of them clear. + pub fn with_pages(pages: u64) -> Self { + Self { words: vec![0; pages.div_ceil(64) as usize], len: pages } + } + + /// How many pages this set is able to describe. Anything at or above it is out of the + /// question being asked, not a member that happens to be absent. + pub fn capacity(&self) -> u64 { + self.len + } + + /// Marks a page. `false` when the page is outside this set, which a caller counts rather + /// than ignores: a reference past the end of the file is a fact worth reporting. + pub fn insert(&mut self, pgno: Pgno) -> bool { + let at = pgno as u64; + if at >= self.len { + return false; + } + self.words[(at / 64) as usize] |= 1 << (at % 64); + true + } + + pub fn contains(&self, pgno: Pgno) -> bool { + let at = pgno as u64; + at < self.len && self.words[(at / 64) as usize] & (1 << (at % 64)) != 0 + } + + /// How many pages are marked. + pub fn count(&self) -> u64 { + self.masked_words().map(|w| w.count_ones() as u64).sum() + } + + /// Pages in neither this set nor `other`. + pub fn count_absent_from_both(&self, other: &PageSet) -> u64 { + // `& mask` on the complement, not on the inputs. Inverting a word turns every bit past + // the end of the file into a page that is absent from both sets, and a file whose page + // count is not a multiple of 64 would report up to 63 leaks that are not pages. + self.pair(other).map(|(a, b, mask)| ((!(a | b)) & mask).count_ones() as u64).sum() + } + + /// Every page in neither set, lowest first. `DoubleEndedIterator`, so the *highest* - the + /// one that pins the tail - is one `next_back` away. + pub fn absent_from_both<'a>( + &'a self, + other: &'a PageSet, + ) -> impl DoubleEndedIterator + 'a { + let pages: Vec = self + .pair(other) + .enumerate() + .flat_map(|(i, (a, b, mask))| { + let free = (!(a | b)) & mask; + (0..64u64).filter_map(move |bit| { + (free & (1 << bit) != 0).then_some((i as u64 * 64 + bit) as Pgno) + }) + }) + .collect(); + pages.into_iter() + } + + /// Pages in both sets, lowest first. + pub fn present_in_both<'a>(&'a self, other: &'a PageSet) -> impl Iterator + 'a { + self.pair(other).enumerate().flat_map(|(i, (a, b, mask))| { + let both = a & b & mask; + (0..64u64).filter_map(move |bit| { + (both & (1 << bit) != 0).then_some((i as u64 * 64 + bit) as Pgno) + }) + }) + } + + /// The words of both sets, each with the mask of the pages that actually exist. + /// + /// The mask is handed out rather than pre-applied because the caller that needs it most is + /// the one taking a *complement*: masking the inputs does nothing for `!(a | b)`, which + /// sets every bit past the end of the file. A file whose page count is not a multiple of + /// 64 would then report up to 63 leaks that are not pages at all. + fn pair<'a>(&'a self, other: &'a PageSet) -> impl Iterator + 'a { + debug_assert_eq!(self.len, other.len, "two page sets over different files"); + let masks = self.masks(); + self.words.iter().zip(other.words.iter()).zip(masks).map(|((a, b), m)| (*a, *b, m)) + } + + /// One mask per word: all ones, except the last, which keeps only the pages that exist. + fn masks(&self) -> impl Iterator + '_ { + let last = (self.len % 64) as u32; + let tail = self.words.len().saturating_sub(1); + (0..self.words.len()).map(move |i| { + if i == tail && last != 0 { + (1u64 << last) - 1 + } else { + u64::MAX + } + }) + } + + fn masked_words(&self) -> impl Iterator + '_ { + self.words.iter().zip(self.masks()).map(|(w, m)| w & m) + } +} + +/// How many pages of each kind the mark phase reached. Every figure counts *marks made*, so a +/// page shared between a snapshot and the current tree is counted under both. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub struct AuditTally { + pub meta: u64, + pub chains: u64, + pub snapshot_chains: u64, + pub trees: u64, + pub snapshot_trees: u64, +} + +/// What an audit found. Every number is exact as of [`LeakReport::txn_id`] and no later: the +/// audit holds one read transaction from start to finish, so a commit that lands while it runs +/// is simply not part of the question it answered. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct LeakReport { + /// The transaction the file was at when the audit began. + pub txn_id: TxnId, + /// `meta.page_count` - the universe of the question. + pub page_count: u64, + /// What the backing file actually holds. Larger than `page_count` means pages the meta + /// page does not account for; see [`LeakReport::beyond_meta`]. + pub file_pages: u64, + /// Pages something points at. + pub reachable: u64, + /// Pages the freelist holds, reusable or not. + pub free_total: u64, + /// The subset of those a writer could take right now. + pub free_reusable: u64, + /// **Pages nothing points at and nothing has recorded as free.** Space the file will never + /// use and never give back. + pub leaked: u64, + /// The highest of them. This is the number that explains a `truncate_tail` that released + /// nothing: every free page below it is pinned by it. + pub highest_leaked: Option, + /// The lowest few, for somebody who wants to go and look at them. + /// + /// **Not an input to anything that frees.** These page numbers were derived under a read + /// transaction while writers were free to commit; a later commit may have put any of them + /// to work. Acting on this list requires re-deriving it under the write lock with no + /// readers alive, the way `Store::truncate_tail` does. + pub leaked_sample: Vec, + /// Pages in the file past what the meta page accounts for. Nonzero after a crash between + /// the meta write and the truncation in `truncate_tail`; those pages are absorbed on the + /// next open and never recovered. + pub beyond_meta: u64, + /// References to pages at or beyond the end of the file. **Always a damaged file.** + pub dangling: u64, + /// Pages both reachable and *reusable*-free - a page that has been handed out twice. + /// + /// Reachable-and-free on its own is ordinary: a page a snapshot pins is reachable from + /// that snapshot's tree and sits in the freelist as a pending run. Restricting to reusable + /// runs removes exactly that case, because a snapshot at transaction `S` holds the horizon + /// at or below `S` while everything its tree reaches was freed after `S`. What is left is + /// corruption, and nothing else in this tree looks for it. + pub double_allocated: u64, + pub double_allocated_sample: Vec, + pub by_class: AuditTally, +} + +impl LeakReport { + /// Whether the file accounts for every page it contains. + pub fn is_clean(&self) -> bool { + self.leaked == 0 + && self.beyond_meta == 0 + && self.dangling == 0 + && self.double_allocated == 0 + } +} + +/// How many pages a run covers, clamped into the file. +pub(crate) fn run_pages(run: &FreeRun, limit: u64) -> impl Iterator { + let first = run.first as u64; + let end = (first + run.len as u64).min(limit); + (first..end).map(|p| p as Pgno) +} diff --git a/crates/big-pager/src/chainio.rs b/crates/big-pager/src/chainio.rs index f4735c5..80ef791 100644 --- a/crates/big-pager/src/chainio.rs +++ b/crates/big-pager/src/chainio.rs @@ -43,6 +43,37 @@ pub fn load_chain( Ok(out) } +/// The same walk as [`chain_pgnos`], with every page's checksum verified before its `next` is +/// followed. +/// +/// **For a caller that is auditing rather than recycling.** `chain_pgnos` trusts `next`, which +/// is right on the write path: it is reading a chain this process just wrote, and a checksum +/// failure there would turn a damaged file into a refused *commit*. An audit is the opposite +/// case - it reads a file it has no reason to trust, and an unverified `next` sends the walk to +/// an arbitrary page which it would then count as live. That under-reports, which is the +/// silent direction, so this one pays for the checksum. +pub fn chain_pgnos_checked( + pager: &P, + root: Option, + ty: PageType, + stride: usize, +) -> Result> { + let mut out = Vec::new(); + let mut cur = root; + let limit = pager.page_count() + 1; + + while let Some(pgno) = cur { + if out.len() as u64 > limit { + return Err(StoreError::ChainCycle { root: root.unwrap_or(0) }); + } + out.push(pgno); + let page = pager.read(pgno)?; + page.verify_checksum()?; + cur = ChainPage::parse(&page, ty, stride)?.next(); + } + Ok(out) +} + /// Walks the chain and returns every page number, needed to hand them back to the freelist /// when the chain is rewritten. pub fn chain_pgnos( diff --git a/crates/big-pager/src/freelist.rs b/crates/big-pager/src/freelist.rs index 75f6412..7b07fbe 100644 --- a/crates/big-pager/src/freelist.rs +++ b/crates/big-pager/src/freelist.rs @@ -85,12 +85,73 @@ impl Freelist { self.runs.push(FreeRun { freed_at, first: pgno, len: 1 }); } + /// Takes back the pages a transaction both allocated and freed, and answers with the + /// `next_pgno` it should commit. + /// + /// **Without this they are lost.** A page a transaction allocates from the tail and then + /// frees never reaches the freelist - it goes to the transaction's scratch list, to be + /// handed straight back out if the same transaction allocates again. A transaction whose + /// tree *shrinks* allocates fewer times than it frees, so what is left over at commit is + /// counted in `page_count`, materialised in the file, and referenced by nothing: a page + /// the database will never read and never reuse. One of those at the end of the file is + /// enough to stop [`crate::Store::truncate_tail`] releasing anything at all. + /// + /// A page flush against the end is given back by lowering `next_pgno` rather than by being + /// recorded as free - the file simply never grows that far. Everything else is pushed. + /// + /// `freed_at` must be the committing transaction's own id, never lower: all three horizon + /// computations are at or below the *base* transaction, so a page stamped with this one is + /// pending for the whole of its own commit and cannot be handed out by the allocations + /// that follow - including the freelist's own, whose termination argument (readme §7.3) + /// assumes the entry count only falls while it runs. + pub fn absorb_scratch( + &mut self, + mut scratch: Vec, + next_pgno: u64, + freed_at: TxnId, + ) -> u64 { + if scratch.is_empty() { + return next_pgno; + } + scratch.sort_unstable(); + // Freeing one page twice inside a transaction is a bug above this line either way, and + // it is already one today: `alloc` would hand the same page out twice. Collapsing it + // here makes the consequence the survivable one - a leak rather than two owners of one + // page - while the assertion still fails the test that produced it. + let before = scratch.len(); + scratch.dedup(); + debug_assert_eq!(before, scratch.len(), "a page was freed twice inside one transaction"); + + let mut next = next_pgno; + while scratch.last().is_some_and(|p| *p as u64 + 1 == next) { + scratch.pop(); + next -= 1; + } + for p in scratch { + self.push(p, freed_at); + } + next + } + /// Merges adjacent runs sharing a `freed_at`. This is where the run-length encoding pays off. + /// + /// **Ordered by page number, and that is what makes the file able to shrink.** It used to + /// be ordered by `freed_at` first, so [`Freelist::alloc`] handed out the oldest generation + /// before the lowest page - which is fine for correctness and hopeless for the tail: holes + /// near the front of the file stayed holes while writes kept landing at the end, and + /// nothing was ever flush against EOF for [`Freelist::trim_tail`] to release. Lowest first, + /// and ordinary writes fill from the front until the tail is free. + /// + /// The reclaim rule is untouched: `alloc` still skips any run newer than the horizon, so + /// this changes *which* reusable page is chosen and never whether one is reusable. The + /// merge is unaffected too - two runs that meet in page order cannot have another between + /// them - and so is the termination argument in `alloc_freelist_pages`, which rests on + /// `alloc` taking from the front of a *run* rather than on the order of the runs. pub fn compact(&mut self) { if self.runs.len() < 2 { return; } - self.runs.sort_unstable(); + self.runs.sort_unstable_by_key(|r| (r.first, r.freed_at)); let mut merged: Vec = Vec::with_capacity(self.runs.len()); for r in self.runs.drain(..) { match merged.last_mut() { @@ -106,7 +167,11 @@ impl Freelist { self.runs = merged; } - /// Takes one reclaimable page. `None` means the file has to grow. + /// Takes one reclaimable page, the lowest there is. `None` means the file has to grow. + /// + /// Lowest because [`Freelist::compact`] leaves the runs in page order, which is what lets + /// the tail empty out and the file shrink. From the **front** of the run it picks, which is + /// what keeps the entry count non-increasing and `alloc_freelist_pages` terminating. pub fn alloc(&mut self, horizon: TxnId) -> Option { let idx = self.runs.iter().position(|r| r.freed_at <= horizon)?; let run = &mut self.runs[idx]; diff --git a/crates/big-pager/src/lib.rs b/crates/big-pager/src/lib.rs index 4cc4f30..43baadf 100644 --- a/crates/big-pager/src/lib.rs +++ b/crates/big-pager/src/lib.rs @@ -27,6 +27,7 @@ #![deny(unsafe_code)] +pub mod audit; pub mod chainio; #[cfg(feature = "counting-pager")] pub mod counting; @@ -44,6 +45,7 @@ pub mod txn; #[cfg(unix)] pub mod mmap; +pub use audit::{AuditTally, LeakReport, PageSet}; #[cfg(feature = "counting-pager")] pub use counting::{CountingPager, PagerCounts}; pub use durability::Durability; @@ -73,7 +75,7 @@ pub use big_page::{ use big_page::{ build_chain, chain_pages_needed as pages_needed, pick_meta, PageType, ROOT_RECORD_BYTES, }; -use chainio::{chain_pgnos, load_chain}; +use chainio::{chain_pgnos, chain_pgnos_checked, load_chain}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex, MutexGuard, RwLock}; @@ -191,6 +193,82 @@ struct StoreState { freelist: Freelist, } +/// An audit in progress: the file's own bookkeeping, captured at one instant, plus the reader +/// that keeps it true while the caller walks the trees. +/// +/// Everything reachable without leaving this crate is already marked. The caller marks the +/// trees - `roots` and then `snapshot_roots`, both of which need `big-btree` - and calls +/// [`Audit::finish`]. +pub struct Audit<'db, P: Pager> { + /// Held for the life of the audit. Dropping it early would let the horizon advance and a + /// page named in the capture be recycled while the walk is still reading it. + _read: ReadTxn<'db, P>, + /// Pages something points at. The caller adds the trees. + pub marks: PageSet, + /// Pages the freelist holds, pending runs included. + pub free: PageSet, + /// Roots of the current trees, for the caller to walk. + pub roots: Vec, + /// Roots reachable only from a snapshot's old roots chain. **The class every other walk in + /// this tree skips**, and the one an audit must not. + pub snapshot_roots: Vec, + /// Runs a writer could take right now. Anything both reachable and in one of these has + /// been handed out twice. + reusable: Vec, + tally: AuditTally, + dangling: u64, + free_total: u64, + free_reusable: u64, + page_count: u64, + file_pages: u64, + txn_id: TxnId, +} + +impl Audit<'_, P> { + /// Marks a page the caller reached, counting it against a tree or a snapshot's tree. + pub fn mark_tree_page(&mut self, pgno: Pgno, from_snapshot: bool) { + if self.marks.insert(pgno) { + if from_snapshot { + self.tally.snapshot_trees += 1; + } else { + self.tally.trees += 1; + } + } else if pgno as u64 >= self.page_count { + self.dangling += 1; + } + } + + /// The complement, once the caller has marked everything it can reach. + pub fn finish(self) -> LeakReport { + const SAMPLE: usize = 64; + let leaked: Vec = self.marks.absent_from_both(&self.free).take(SAMPLE).collect(); + let mut reusable = PageSet::with_pages(self.page_count); + for run in &self.reusable { + for p in audit::run_pages(run, self.page_count) { + reusable.insert(p); + } + } + let doubled: Vec = self.marks.present_in_both(&reusable).take(SAMPLE).collect(); + + LeakReport { + txn_id: self.txn_id, + page_count: self.page_count, + file_pages: self.file_pages, + reachable: self.marks.count(), + free_total: self.free_total, + free_reusable: self.free_reusable, + leaked: self.marks.count_absent_from_both(&self.free), + highest_leaked: self.marks.absent_from_both(&self.free).next_back(), + leaked_sample: leaked, + beyond_meta: self.file_pages.saturating_sub(self.page_count), + dangling: self.dangling, + double_allocated: self.marks.present_in_both(&reusable).count() as u64, + double_allocated_sample: doubled, + by_class: self.tally, + } + } +} + /// Bound only by `Pager`/`PagerMut`, so swapping the backend touches nothing in here. pub struct Store { pager: P, @@ -326,6 +404,137 @@ impl Store

{ Ok(ReadTxn::new(self, snap.txn_id, Arc::new(roots), catalog)) } + /// Opens an audit: a reader, plus everything about the file's own bookkeeping taken at the + /// same instant. + /// + /// **The capture has to happen here, under one lock.** A caller assembling the same facts + /// from `meta()`, `snapshots()` and the freelist would take three separate read guards, and + /// `commit_txn` publishes all of them under one write guard - so a mark set built from a + /// straddled state would be wrong in a way no test reliably catches. + /// + /// This marks everything it can reach without leaving this crate: the meta pages, the four + /// chains, and every chain a snapshot still names. What it cannot do is walk a b-tree - + /// `big-btree` depends on this crate, not the other way round - so it hands back the roots + /// and leaves that to the caller. See `Db::audit_pages`. + /// + /// The read transaction is held for the whole audit, which pins the reclaim horizon just as + /// a backup does: `truncate_tail` will refuse for the duration, and any page named in this + /// capture is safe from being recycled underneath the walk. + pub fn begin_audit(&self) -> Result> { + // One guard. The reader is registered before it is dropped, so nothing committed + // between the capture and the registration can move the horizon past us. + let (read, meta, freelist, snapshots, roots, horizon) = { + let st = self.state.read().unwrap(); + let reader_h = self.oldest_reader().unwrap_or(st.meta.txn_id); + let snap_h = st.snapshots.oldest_txn_id().unwrap_or(st.meta.txn_id); + let read = + ReadTxn::new(self, st.meta.txn_id, Arc::clone(&st.roots), Arc::clone(&st.catalog)); + ( + read, + st.meta, + st.freelist.clone(), + Arc::clone(&st.snapshots), + Arc::clone(&st.roots), + reader_h.min(snap_h).min(st.meta.txn_id), + ) + }; + + let pages = meta.page_count; + let mut marks = PageSet::with_pages(pages); + let mut free = PageSet::with_pages(pages); + let mut tally = AuditTally::default(); + let mut dangling = 0u64; + + // 1. Both meta slots. They are double-buffered - a commit writes only `txn_id % 2` - + // so the one this file is not currently governed by still holds the previous commit + // and is what `Store::load` falls back to. + for slot in 0..META_PAGES { + if marks.insert(slot as Pgno) { + tally.meta += 1; + } + } + + // 2. The four chains' own pages. + for (head, ty, stride) in Self::chain_shapes(&meta) { + for p in chain_pgnos_checked(&self.pager, head, ty, stride)? { + if marks.insert(p) { + tally.chains += 1; + } else { + dangling += 1; + } + } + } + + // 3. Every snapshot's *old* roots chain, and the roots on it. This is the class no + // other walk in the tree visits: those trees are unreachable from the current roots + // and entirely live. `0` is the absent marker, the same one `begin_read_at` uses. + let mut snapshot_roots = Vec::new(); + for snap in snapshots.all() { + let head = (snap.root_records != 0).then_some(snap.root_records); + if head.is_none() { + continue; + } + for p in + chain_pgnos_checked(&self.pager, head, PageType::RootRecords, ROOT_RECORD_BYTES)? + { + if marks.insert(p) { + tally.snapshot_chains += 1; + } else { + dangling += 1; + } + } + let old = RootRecords::from_entries(&load_chain( + &self.pager, + head, + PageType::RootRecords, + ROOT_RECORD_BYTES, + )?); + snapshot_roots.extend(old.iter().map(|(_, p)| *p)); + } + + // 4. Everything the freelist holds, pending runs included: pending is not reusable, + // but it is certainly not unaccounted for. + let mut free_total = 0u64; + let mut reusable = Vec::new(); + for run in freelist.runs() { + for p in audit::run_pages(run, pages) { + if free.insert(p) { + free_total += 1; + } + } + if run.freed_at <= horizon { + reusable.push(*run); + } + } + + Ok(Audit { + _read: read, + marks, + free, + roots: roots.iter().map(|(_, p)| *p).collect(), + snapshot_roots, + reusable, + tally, + dangling, + free_total, + free_reusable: freelist.reusable_pages(horizon), + page_count: pages, + file_pages: self.pager.page_count(), + txn_id: meta.txn_id, + }) + } + + /// The four chains, as `(head, type, stride)`. One list, so a caller cannot walk three of + /// them and forget the fourth. + fn chain_shapes(meta: &MetaPage) -> [(Option, PageType, usize); 4] { + [ + (meta.root_records, PageType::RootRecords, ROOT_RECORD_BYTES), + (meta.freelist, PageType::Freelist, FREE_ENTRY_BYTES), + (meta.snapshots, PageType::Snapshots, SNAPSHOT_ENTRY_BYTES), + (meta.catalog, PageType::Catalog, CATALOG_ENTRY_BYTES), + ] + } + pub(crate) fn register_reader(&self, txn_id: TxnId) { *self.readers.lock().unwrap().entry(txn_id).or_insert(0) += 1; } @@ -622,6 +831,14 @@ impl Store

{ txn_id, )?; let free_pgnos = alloc_freelist_pages(&mut freelist, horizon, next_pgno, cap)?; + // The freelist is about to be written down as the file's record of what is spare, so + // nothing in it may name a page beyond where the file ends. Guards the old invariant as + // much as `absorb_scratch`, which is the newest way to reach it. + debug_assert!( + freelist.runs().iter().all(|r| r.first as u64 + r.len as u64 <= *next_pgno), + "a free run runs past the end of the file: {:?} against {next_pgno}", + freelist.runs() + ); let free_entries = freelist.encode(); // 4. Collect every page that has to reach the disk. A kept chain contributes none. diff --git a/crates/big-pager/src/txn.rs b/crates/big-pager/src/txn.rs index 679334f..040b7f4 100644 --- a/crates/big-pager/src/txn.rs +++ b/crates/big-pager/src/txn.rs @@ -249,7 +249,8 @@ impl<'db, P: PagerMut> WriteTxn<'db, P> { pub fn free(&mut self, pgno: Pgno) { // A page from beyond the tail floor was allocated by this transaction, so it cannot be - // in any committed tree and no reader can be looking at it. + // in any committed tree and no reader can be looking at it. It goes to `scratch`, which + // `alloc` empties first and `commit` gives back - see `Freelist::absorb_scratch`. if pgno as u64 >= self.tail_floor { self.dirty.remove(&pgno); self.scratch.push(pgno); @@ -338,6 +339,15 @@ impl<'db, P: PagerMut> WriteTxn<'db, P> { } pub fn commit(mut self) -> Result { + // **Before anything below allocates.** Nothing in `commit_txn` knows `scratch` exists, + // which is exactly how a transaction whose tree shrank used to lose its own churn into + // `page_count`. It has to happen here rather than deeper in: lowering `next_pgno` after + // a chain page had been taken from the tail would hand that page's number out twice. + self.next_pgno = self.freelist.absorb_scratch( + core::mem::take(&mut self.scratch), + self.next_pgno, + self.txn_id, + ); let txn_id = self.store.commit_txn( self.txn_id, self.base, diff --git a/crates/big-pager/tests/audit.rs b/crates/big-pager/tests/audit.rs new file mode 100644 index 0000000..ec1be8a --- /dev/null +++ b/crates/big-pager/tests/audit.rs @@ -0,0 +1,228 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Accounting for every page: reachable, free, or neither. +//! +//! **The test that matters most here is the positive control.** A leak counter that returns +//! zero is indistinguishable from one that never looks, so `an_orphaned_page_is_reported` - +//! which deliberately strands a page and demands the counter find it - is what makes every +//! other zero in this file mean something. +//! +//! Everything below drives the pager directly rather than a `Db`, because the trees are the +//! caller's half of the walk. The `Db` half is `crates/big-db/tests/leaks.rs`. + +use big_page::{FragmentKey, LeafBuilder}; +use big_pager::*; + +fn key(field: u32, shard: u64) -> FragmentKey { + FragmentKey::new(1, field, 0, shard) +} + +/// One leaf page, named by a root. Returns its page number. +fn put_fragment(store: &Store

, k: FragmentKey) -> u32 { + let mut w = store.begin_write(); + let p = w.alloc().unwrap(); + w.write(p, LeafBuilder::new().finish(p)).unwrap(); + w.set_root(k, p); + w.commit().unwrap(); + p +} + +/// Runs an audit, marking every root's single leaf page - which is the whole of a tree in +/// these fixtures. The real walk is `Db::audit_pages`. +fn audit(store: &Store

) -> LeakReport { + let mut a = store.begin_audit().expect("the file is readable"); + for root in core::mem::take(&mut a.roots) { + a.mark_tree_page(root, false); + } + for root in core::mem::take(&mut a.snapshot_roots) { + a.mark_tree_page(root, true); + } + a.finish() +} + +// ------------------------------------------------------------------------------------------- +// The bitset +// ------------------------------------------------------------------------------------------- + +#[test] +fn marking_the_same_page_twice_counts_once() { + let mut s = PageSet::with_pages(100); + assert!(s.insert(7)); + assert!(s.insert(7)); + assert_eq!(s.count(), 1); + assert!(s.contains(7)); + assert!(!s.contains(8)); +} + +/// A page past the end of the file is not a member that happens to be absent - it is outside +/// the question, and the caller is told so rather than left to assume it landed. +#[test] +fn a_page_past_the_end_is_refused_rather_than_dropped() { + let mut s = PageSet::with_pages(64); + assert!(!s.insert(64)); + assert!(!s.insert(1_000)); + assert_eq!(s.count(), 0); +} + +/// **The off-by-one that will actually happen.** A file whose page count is not a multiple of +/// 64 has bits in its last word describing pages that do not exist, and a complement that +/// counted them would report leaks which are not pages at all. +#[test] +fn the_complement_ignores_the_bits_past_the_end_of_the_file() { + let pages = 70; // not a multiple of 64 + let mut marks = PageSet::with_pages(pages); + let free = PageSet::with_pages(pages); + for p in 0..pages { + marks.insert(p as u32); + } + assert_eq!(marks.count(), pages); + assert_eq!(marks.count_absent_from_both(&free), 0, "six phantom pages in the last word"); + + marks = PageSet::with_pages(pages); + assert_eq!( + marks.count_absent_from_both(&free), + pages, + "and all of them when nothing is marked" + ); +} + +#[test] +fn the_highest_unaccounted_page_is_the_one_that_pins_the_tail() { + let pages = 200; + let free = PageSet::with_pages(pages); + let mut marks = PageSet::with_pages(pages); + // Everything marked but two holes. The higher one is what a truncation walking down from + // the end of the file meets first, and so the one that matters. + for p in 0..pages { + if p != 40 && p != 150 { + marks.insert(p as u32); + } + } + assert_eq!(marks.count_absent_from_both(&free), 2); + assert_eq!(marks.absent_from_both(&free).next_back(), Some(150)); + assert_eq!(marks.absent_from_both(&free).next(), Some(40)); +} + +// ------------------------------------------------------------------------------------------- +// The audit +// ------------------------------------------------------------------------------------------- + +/// A file that has only ever been written to accounts for every page it has. +#[test] +fn a_fresh_store_accounts_for_every_page() { + let store = Store::init(MemPager::new()).unwrap(); + let r = audit(&store); + assert!(r.is_clean(), "{r:?}"); + assert_eq!(r.leaked, 0); + assert_eq!(r.by_class.meta, 2, "both meta slots, always"); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +#[test] +fn a_store_with_fragments_accounts_for_every_page() { + let store = Store::init(MemPager::new()).unwrap(); + for i in 0..16 { + put_fragment(&store, key(1, i)); + } + let r = audit(&store); + assert_eq!(r.leaked, 0, "{r:?}"); + assert_eq!(r.by_class.trees, 16); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +/// **The positive control.** A page is allocated, written, and then named by nothing: not a +/// root, not a chain, and never freed. Without this test every zero above could be a counter +/// that simply never looks. +#[test] +fn an_orphaned_page_is_reported() { + let store = Store::init(MemPager::new()).unwrap(); + put_fragment(&store, key(1, 0)); + + let stranded = { + let mut w = store.begin_write(); + let p = w.alloc().unwrap(); + w.write(p, LeafBuilder::new().finish(p)).unwrap(); + // No `set_root`, no `free`. The page exists and nothing will ever refer to it again. + w.commit().unwrap(); + p + }; + + let r = audit(&store); + assert_eq!(r.leaked, 1, "{r:?}"); + assert_eq!(r.highest_leaked, Some(stranded)); + assert_eq!(r.leaked_sample, vec![stranded]); + assert!(!r.is_clean()); + assert_eq!(r.reachable + r.free_total + r.leaked, r.page_count, "{r:?}"); +} + +/// Every page a snapshot pins is reachable, even though the current roots have forgotten it. +/// +/// **This is the class no other walk in the tree visits** - `copy_to` skips it deliberately +/// and `scrub` skips it by omission. Without it the audit would report every snapshot-pinned +/// page as leaked, and nobody would find out until a file had a snapshot in it. +#[test] +fn a_tree_only_a_snapshot_still_names_is_reachable() { + let store = Store::init(MemPager::new()).unwrap(); + for i in 0..8 { + put_fragment(&store, key(1, i)); + } + + let mut w = store.begin_write(); + w.create_snapshot(u64::MAX, "before", true); + w.commit().unwrap(); + + // Every fragment goes. Their pages are pending-free and reachable only from the snapshot's + // old roots chain. + for i in 0..8 { + let mut w = store.begin_write(); + let root = w.remove_root(&key(1, i)).expect("still there"); + w.free(root); + w.commit().unwrap(); + } + + let r = audit(&store); + assert_eq!(r.leaked, 0, "a snapshot's trees were counted as leaked: {r:?}"); + assert_eq!(r.by_class.snapshot_trees, 8, "{r:?}"); + assert!(r.by_class.snapshot_chains > 0, "the old roots chain itself: {r:?}"); + assert_eq!(r.double_allocated, 0, "pinned pages are pending-free, never reusable: {r:?}"); +} + +/// An audit reads. It must leave the file at exactly the transaction it found it at. +#[test] +fn an_audit_never_changes_the_file() { + let store = Store::init(MemPager::new()).unwrap(); + for i in 0..8 { + put_fragment(&store, key(1, i)); + } + let before = store.metrics(); + let r = audit(&store); + let after = store.metrics(); + + assert_eq!(before.txn_id, after.txn_id); + assert_eq!(before.page_count, after.page_count); + assert_eq!(before.free_pages_reusable, after.free_pages_reusable); + assert_eq!(r.txn_id, before.txn_id); +} + +/// The reader an audit holds is released when it finishes, or the next truncation would refuse +/// for ever. +#[test] +fn an_audit_releases_its_reader() { + let store = Store::init(MemPager::new()).unwrap(); + put_fragment(&store, key(1, 0)); + let _ = audit(&store); + assert_eq!(store.metrics().live_readers, 0); + assert!(store.truncate_tail().is_ok(), "the audit's reader outlived the audit"); +} diff --git a/crates/big-pager/tests/reclamation.rs b/crates/big-pager/tests/reclamation.rs index 5484050..51cdd80 100644 --- a/crates/big-pager/tests/reclamation.rs +++ b/crates/big-pager/tests/reclamation.rs @@ -205,6 +205,129 @@ fn truncate_tail_refuses_while_a_reader_is_alive() { assert!(store.truncate_tail().is_ok()); } +/// **The lowest reusable page is the one handed out**, which is what lets a file that has +/// churned ever get smaller. +/// +/// Ordering the freelist by generation first was correct and hopeless for the tail: writes went +/// on landing at the end of the file while holes near the front stayed holes, so nothing was +/// ever flush against EOF for `truncate_tail` to release. The reclaim rule is untouched - a run +/// newer than the horizon is still skipped - and only the choice among reusable pages changed. +/// Against the freelist itself rather than through a commit, and for the reason this file's +/// header gives about every other assertion here: a commit takes pages for its own root, +/// catalog and freelist chains before the caller's `alloc` ever sees the list, so a page number +/// observed through one says as much about the machinery as about the rule. The rule is a +/// property of this data structure, and this is where it is exact. +#[test] +fn the_lowest_reusable_page_is_allocated_first() { + let mut f = Freelist::default(); + // Freed newest-page-first, so the *oldest* generation holds the *highest* page - which is + // what the old ordering, by generation, would have handed out first. + f.push(30, 1); + f.push(20, 2); + f.push(10, 3); + f.compact(); + + assert_eq!(f.alloc(9), Some(10), "the lowest hole, not the oldest one"); + assert_eq!(f.alloc(9), Some(20)); + assert_eq!(f.alloc(9), Some(30)); + assert_eq!(f.alloc(9), None); +} + +/// And the horizon still decides *whether* a page may be reused, not just which one. A run +/// newer than the horizon is skipped however low it sits. +#[test] +fn a_page_too_new_to_reuse_is_skipped_however_low_it_is() { + let mut f = Freelist::default(); + f.push(10, 7); // low, but freed by a transaction a reader can still see + f.push(30, 1); + f.compact(); + + assert_eq!(f.alloc(3), Some(30), "the low page is not reusable yet"); + assert_eq!(f.alloc(3), None); + assert_eq!(f.alloc(7), Some(10), "and is once the horizon passes it"); +} + +/// And the point of that ordering: a file that churns comes back down instead of only ever +/// growing. Written and dropped repeatedly, then trimmed - the tail is free by then, because +/// every write went into a hole near the front rather than onto the end. +#[test] +fn a_file_that_churns_can_be_given_back_to_the_filesystem() { + let store = Store::init(MemPager::new()).unwrap(); + for i in 0..16 { + put_fragment(&store, key(i, 0)); + } + let high_water = store.metrics().page_count; + + for i in 0..16 { + drop_fragment(&store, key(i, 0)); + } + // A few more commits, which is what a live database is doing while this happens: each one + // takes its pages from the freelist, lowest first, and leaves the tail alone. + for i in 0..4 { + put_fragment(&store, key(100 + i, 0)); + } + + let released = store.truncate_tail().unwrap(); + assert!(released > 0, "the tail was free and none of it was given back"); + assert!( + store.metrics().page_count < high_water, + "the file did not shrink: {} vs {high_water}", + store.metrics().page_count + ); +} + +// ------------------------------------------------------------------------------------------- +// Pages a transaction allocated and then gave up on +// +// A page freed above the tail floor goes to the transaction's scratch list rather than the +// freelist, so `alloc` can hand it straight back. What is left there at commit used to be lost: +// counted in `page_count`, present in the file, and referenced by nothing. Against the freelist +// directly, for the reason at the top of this file. +// ------------------------------------------------------------------------------------------- + +/// A page flush against the end is given back by not growing the file that far, which is +/// better than recording it as free: there is nothing to record and nothing to reuse. +#[test] +fn scratch_against_the_tail_is_given_back_rather_than_freed() { + let mut f = Freelist::default(); + assert_eq!(f.absorb_scratch(vec![10, 11], 12, 5), 10); + assert!(f.is_empty(), "the file simply never grew that far"); +} + +/// A page with something live above it cannot be given back that way, so it is recorded free. +#[test] +fn scratch_below_the_tail_goes_to_the_freelist() { + let mut f = Freelist::default(); + assert_eq!(f.absorb_scratch(vec![10], 12, 5), 12, "page 11 is live, so 10 cannot be dropped"); + assert_eq!(f.runs(), [FreeRun { freed_at: 5, first: 10, len: 1 }]); +} + +/// The giveback stops at the first gap - everything below the gap is recorded instead. +#[test] +fn the_giveback_stops_at_the_first_gap() { + let mut f = Freelist::default(); + assert_eq!(f.absorb_scratch(vec![100, 102], 103, 7), 102); + assert_eq!(f.runs(), [FreeRun { freed_at: 7, first: 100, len: 1 }]); +} + +/// **Stamped with the committing transaction, so it is pending for the whole of that commit.** +/// A lower stamp would make the page allocatable while the freelist's own pages are still being +/// chosen, which is the loop whose termination argument assumes the entry count only falls. +#[test] +fn an_absorbed_page_is_not_reusable_inside_its_own_commit() { + let mut f = Freelist::default(); + f.absorb_scratch(vec![10], 12, 5); + assert_eq!(f.alloc(4), None, "the horizon is below this transaction"); + assert_eq!(f.alloc(5), Some(10), "and it is reusable once the horizon reaches it"); +} + +#[test] +fn absorbing_nothing_changes_nothing() { + let mut f = Freelist::default(); + assert_eq!(f.absorb_scratch(Vec::new(), 12, 5), 12); + assert!(f.is_empty()); +} + #[test] fn readers_are_refcounted_not_flagged() { let store = Store::init(MemPager::new()).unwrap(); diff --git a/crates/big-pager/tests/store.rs b/crates/big-pager/tests/store.rs index e9a7bc6..8c89fb1 100644 --- a/crates/big-pager/tests/store.rs +++ b/crates/big-pager/tests/store.rs @@ -266,6 +266,61 @@ fn backend_is_swappable() { } } +/// **A transaction that gives its own pages back must leave the file the size it would have +/// been.** Two stores, the same committed content; one of them additionally takes five pages +/// from the tail and frees them all inside the same transaction. Those pages are in no tree, in +/// no chain, and no reader ever saw them - a file that ends up larger for them has kept pages +/// nothing will ever read and nothing will ever hand out again. +/// +/// A difference test rather than an absolute one, so it stays true when the machinery around it +/// changes how many pages a commit needs. +#[test] +fn a_transaction_that_abandons_its_own_pages_leaves_no_trace() { + let plain = Store::init(MemPager::new()).unwrap(); + put_fragment(&plain, key(1, 0)); + + let churned = Store::init(MemPager::new()).unwrap(); + put_fragment(&churned, key(1, 0)); + let mut w = churned.begin_write(); + let pages: Vec = (0..5).map(|_| w.alloc().unwrap()).collect(); + for p in pages { + w.free(p); + } + w.commit().unwrap(); + // The plain store commits too, so both have run the same number of transactions. + plain.begin_write().commit().unwrap(); + + assert_eq!( + churned.pager().page_count(), + plain.pager().page_count(), + "five pages allocated and freed in one transaction were kept" + ); +} + +/// The plateau below, but with a transaction that churns. **This is the one the leak broke:** +/// an empty commit has nothing in its scratch list, so it plateaus either way. +#[test] +fn repeated_churning_commits_do_not_grow_the_file_without_bound() { + let store = Store::init(MemPager::new()).unwrap(); + put_fragment(&store, key(1, 0)); + let churn = || { + let mut w = store.begin_write(); + let pages: Vec = (0..3).map(|_| w.alloc().unwrap()).collect(); + for p in pages { + w.free(p); + } + w.commit().unwrap(); + }; + for _ in 0..50 { + churn(); + } + let settled = store.pager().page_count(); + for _ in 0..200 { + churn(); + } + assert_eq!(store.pager().page_count(), settled, "page count must plateau under churn"); +} + /// Freelist pages always come from the file tail, so a quiet loop of commits must still /// reach a steady state rather than growing the file forever. #[test] diff --git a/crates/big-proxy/tests/config.rs b/crates/big-proxy/tests/config.rs index 3d88458..476bece 100644 --- a/crates/big-proxy/tests/config.rs +++ b/crates/big-proxy/tests/config.rs @@ -39,7 +39,7 @@ fn both_readers_agree_on_the_shipped_cluster_file() { let proxy_pairs: Vec<(String, String)> = proxy.into_iter().map(|n| (n.name, n.addr)).collect(); assert_eq!(daemon_pairs, proxy_pairs, "the readers disagree about who is in the cluster"); - assert_eq!(daemon_pairs.len(), 3, "the shipped example is three nodes"); + assert_eq!(daemon_pairs.len(), 2, "the shipped example is two nodes, a range each"); } /// The divergence, pinned. @@ -101,3 +101,93 @@ fn neither_reader_accepts_a_file_with_no_nodes() { assert!(ClusterFile::parse(empty).is_err()); assert!(parse_cluster(empty, "empty").is_err()); } + +// ------------------------------------------------------------------------------------------- +// The Kubernetes manifests +// ------------------------------------------------------------------------------------------- +// +// `deploy/k8s` ships two TOML documents inside one ConfigMap, and they are not +// interchangeable - which is exactly the mistake that is easy to make while editing YAML. The +// tests below pin the difference: +// +// - `seed.toml` is a **membership**: a node with no log counts a majority out of it, so it must +// satisfy the daemon's reader, ranges and all. +// - `upstreams.toml` is an **address space**: it deliberately names pods that do not exist yet, +// so that `kubectl scale` reaches a client with no proxy restart. Only this crate's reader +// ever sees it, and it must accept it. +// +// A third file exists only at runtime: `join.sh` writes the cluster file a joining pod starts +// with, by appending a `replica` entry per name up to its own ordinal. That append is +// reproduced here, because a file the daemon refuses is a pod that crash-loops with no clue +// pointing back at a shell script in a ConfigMap. + +/// One `key: |` block scalar out of a ConfigMap, dedented. +/// +/// A parser rather than a YAML dependency: what is needed is one block, the indentation is +/// fixed at four spaces by the file itself, and a test that pulled in a YAML crate to read two +/// strings would be a dependency the shipped binary is measured against. +fn block(yaml: &str, key: &str) -> String { + let opener = format!(" {key}: |"); + let start = yaml.find(&opener).unwrap_or_else(|| panic!("no `{key}:` block in the manifest")); + let body = &yaml[start + opener.len()..]; + let mut out = String::new(); + for line in body.lines().skip(1) { + if !line.is_empty() && !line.starts_with(" ") { + break; + } + out.push_str(line.strip_prefix(" ").unwrap_or("")); + out.push('\n'); + } + out +} + +fn manifest() -> String { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../deploy/k8s/10-config.yaml"); + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display())) +} + +#[test] +fn the_k8s_seed_is_a_membership_both_readers_accept() { + let seed = block(&manifest(), "seed.toml"); + let daemon = ClusterFile::parse(&seed).expect("the seed is what a founding node starts from"); + let proxy = parse_cluster(&seed, "seed").expect("and the proxy could read it too"); + + let names: Vec<&str> = daemon.nodes().iter().map(|n| n.name.as_str()).collect(); + assert_eq!(names, ["big-0", "big-1", "big-2"], "three, because a majority of two is two"); + assert_eq!(proxy.len(), 3); + assert_eq!(proxy[0].addr, "big-0.nodes:7654", "the address is the headless Service's"); + assert_ne!(proxy[0].name, proxy[0].addr, "the name is what the certificate carries"); +} + +#[test] +fn the_k8s_upstream_list_may_name_pods_that_do_not_exist() { + let upstreams = block(&manifest(), "upstreams.toml"); + let proxy = parse_cluster(&upstreams, "upstreams").expect("a front door does not vote"); + assert!(proxy.len() > 3, "the ceiling is above the founding size, or scaling needs a restart"); + + // And the daemon must refuse it, which is why it is a separate file: every entry past the + // third has no range, so this is not a shard map and was never meant to be seeded from. + assert!( + ClusterFile::parse(&upstreams).is_err(), + "if the daemon accepts this, somebody has given the upstream list shard ranges - and a \ + node seeded from it would count a majority out of pods that do not exist" + ); +} + +#[test] +fn a_joining_pods_generated_file_is_one_the_daemon_accepts() { + // What `join.sh` writes for `big-4`: the seed, then every name up to its own ordinal that + // the seed does not already have, as a replica of the first node. + let mut file = block(&manifest(), "seed.toml"); + for i in 3..=4 { + file.push_str(&format!( + "\n[[node]]\nname = \"big-{i}\"\naddr = \"big-{i}.nodes:7654\"\nreplica = \"big-0\"\n" + )); + } + + let daemon = ClusterFile::parse(&file).expect("a joining node must be able to start"); + assert_eq!(daemon.nodes().len(), 5); + // The roster is the point: the leader that dials this node may be any of them. + let names: Vec<&str> = daemon.nodes().iter().map(|n| n.name.as_str()).collect(); + assert!(names.contains(&"big-0") && names.contains(&"big-4")); +} diff --git a/deploy/cluster/certs.sh b/deploy/cluster/certs.sh index 66364a6..f38e126 100755 --- a/deploy/cluster/certs.sh +++ b/deploy/cluster/certs.sh @@ -11,16 +11,23 @@ set -eu cd "$(dirname "$0")" -nodes="a b a-spare" +# **The names, and where they are written**, both overridable so that a second deployment can +# have a CA of its own without a second copy of this script. `deploy/k8s` uses both: its nodes +# are a StatefulSet's pods, so they are called `big-0`, `big-1`, and it keeps its own `secrets/` +# because two deployments sharing one CA is two deployments that can impersonate each other. +# +# NODES="big-0 big-1 big-2" SECRETS=../k8s/secrets ./certs.sh +nodes=${NODES:-"a b"} +out=${SECRETS:-secrets} days=${DAYS:-365} -mkdir -p secrets -chmod 700 secrets -cd secrets +mkdir -p "$out" +chmod 700 "$out" +cd "$out" # **With a name, this issues one more certificate and leaves everything else alone.** A node that # joins a running cluster needs one, and the alternative - re-running the whole script - would -# rotate the CA and lock out the three nodes that are already talking. Called with no arguments it +# rotate the CA and lock out the nodes that are already talking. Called with no arguments it # is the bootstrap it always was, and refuses to run twice for the same reason. # # ./certs.sh the CA and a certificate for each of the nodes above diff --git a/deploy/cluster/cluster.toml b/deploy/cluster/cluster.toml index db102ea..2523c99 100644 --- a/deploy/cluster/cluster.toml +++ b/deploy/cluster/cluster.toml @@ -1,8 +1,9 @@ -# Three nodes: two ranges and a copy of one of them. +# Two nodes: a range each, and no copy of either. # -# Three is the minimum once anything has a copy, and `big serve` refuses fewer: failing over is a -# decision a majority has to agree on, and a majority of two is two. `b` counts towards that -# majority even though it holds a different range. +# Nothing here is replicated, so nothing fails over - a node that stops takes its range with it +# until it comes back. A copy would need a third node and not a second: failing over is a +# decision a majority has to agree on, and a majority of two is two, so two machines can never +# use a copy they hold. See docs/clustering.md, "Replication, and failing over". # # The addresses are the compose service names. Inside the network each node listens on # 0.0.0.0:7654 and reaches the others by name, which is why each one is started with --node: @@ -44,10 +45,3 @@ shards = "0..64" name = "b" addr = "b:7654" shards = "64.." - -# A copy of everything `a` holds. No range of its own: it takes `a`'s. If the node serving that -# range stops answering, the agreement gives the range to this one. -[[node]] -name = "a-spare" -addr = "a-spare:7654" -replica = "a" diff --git a/deploy/cluster/docker-compose.yml b/deploy/cluster/docker-compose.yml index f57dfb6..dc88ba0 100644 --- a/deploy/cluster/docker-compose.yml +++ b/deploy/cluster/docker-compose.yml @@ -1,5 +1,5 @@ -# Three nodes, one of them a copy. See cluster.toml for the shape and docs/clustering.md for -# why it is that shape. +# Two nodes, a range each. See cluster.toml for the shape and docs/clustering.md for why it is +# that shape. # # cd deploy/cluster # ./users.sh # writes secrets/users, mode 600, and prints the passwords @@ -19,9 +19,34 @@ # So the published port is `proxy` now. A client still sees one address; it is no longer one # machine. # -# `stop a` and the queries keep working. That is the whole difference. +# **Nothing here is replicated, and that is a choice with a price.** Each range lives on exactly +# one node, so `stop b` leaves `64..` unanswerable until it comes back: the proxy picks a node +# that is up, it cannot pick data that is not there. Failing over needs a copy of a range, and a +# copy needs a third node to settle the vote - see docs/clustering.md, "Replication, and failing +# over", and deploy/readme.md, "What a joining node needs first", for what adds one. +# +# **Three things a node can now do for itself, and only one of them is on here.** Each is a flag +# and each is off by default, for one reason: a deployment that reshapes itself unasked is one +# whose shape an operator cannot predict. +# +# --reclaim on below. Purely local - it asks no peer and proposes nothing, so it +# runs on every node, clustered or not, leader or not. See the flag. +# --balance off. It works, and it moves the facts rather than only the map: a +# range handed over is seeded, caught up, cut over behind a write +# barrier, checked by digest, and only then dropped from the source. +# Off because the two ranges here are the thing this file is +# explaining, and a cluster that redraws them mid-demo is explaining +# something else. It also needs both nodes up to decide anything, for +# the reason below. +# --elect-schema-leader off, and here it would be **inert rather than merely cautious**. It +# hands the row-key namespace on after fifteen seconds of silence from +# whoever holds it, and handing it on is a decision the agreement has +# to commit. A majority of two is two - so at the moment the schema +# leader is the node that went away, there is no majority left to +# commit anything with. This flag wants the same third node that +# replication wants. -# Everything the three nodes share. Not the volumes: a merge key replaces a list rather than +# Everything the two nodes share. Not the volumes: a merge key replaces a list rather than # extending it, so a `volumes` here would be silently overridden by each service's own and read # as though it applied. x-node: &node @@ -73,7 +98,7 @@ services: # Staged from the mount by the entrypoint, which reads it as root and writes a private # copy: a bind mount carries the host's ownership, which the image cannot predict. - /run/big/users - # What this node presents to the other two. The CA that checks it is named in cluster.toml, + # What this node presents to the other node. The CA that checks it is named in cluster.toml, # because that half is the same everywhere; these two are this node's alone. - --peer-cert - /run/big/peer.pem @@ -84,6 +109,16 @@ services: # ceiling is a daemon one expensive query can hold a worker on for as long as it likes. - --query-timeout - "30" + # Trailing free pages go back to the filesystem while this node serves. Local, and gated + # on nothing but this switch: no peer is asked and nothing is proposed, so it does not + # care that this is a cluster. It waits for the file to reach 4096 pages with a quarter of + # it reusable, which a demo this small never reaches - it is here because this file is + # also a template. **What it gives back is the free run at the end of the file and nothing + # else**: a database that has churned in the middle keeps those holes until an offline + # `big compact` rewrites it, because moving a live page means rewriting whoever points at + # it. `big leaks ` prints the accounting, offline. Watch big_pages_reclaimed_total + # and big_reclaim_blocked_total; the second one climbing means readers, not a fault. + - --reclaim # **The same certificate again, and this is the pair that makes the cluster work.** # `--peer-cert` is the identity this node presents when it *dials* another; `--tls-cert` is # what its own listener presents when it is dialled. A node given only the first speaks TLS @@ -122,56 +157,7 @@ services: # Staged from the mount by the entrypoint, which reads it as root and writes a private # copy: a bind mount carries the host's ownership, which the image cannot predict. - /run/big/users - # What this node presents to the other two. The CA that checks it is named in cluster.toml, - # because that half is the same everywhere; these two are this node's alone. - - --peer-cert - - /run/big/peer.pem - - --peer-key - - /run/big/peer.key - # **Set it on a new deployment.** There is no default, on purpose - a query that used to - # finish must not start failing because the server was upgraded - but a daemon with no - # ceiling is a daemon one expensive query can hold a worker on for as long as it likes. - - --query-timeout - - "30" - # **The same certificate again, and this is the pair that makes the cluster work.** - # `--peer-cert` is the identity this node presents when it *dials* another; `--tls-cert` is - # what its own listener presents when it is dialled. A node given only the first speaks TLS - # outward into a plaintext port and every peer handshake fails - `received corrupt message - # of type InvalidContentType`, an election that never settles, `serving: false`. One - # listener serves peers and people both, so there is one certificate for both, and - # `certs.sh` issues it with `serverAuth` and `clientAuth` for exactly this reason. - # - # It also means the published port is HTTPS: `bigctl --addr https://... --ca-file` rather - # than a bare host:port. Client certificates are optional on this listener, so a person - # with no certificate still connects and still authenticates with a password. - - --tls-cert - - /run/big/peer.pem - - --tls-key - - /run/big/peer.key - # Per node, because a backup is per node: each holds its own range, and a copy of one node - # is a copy of one range. `big backup` cannot do this while the daemon runs - it wants the - # exclusive lock the daemon holds - so `POST /admin/backup` is the online path. - - --backup-dir - - /data - - a-spare: - <<: *node - volumes: - - spare-data:/data - - ./cluster.toml:/etc/big/cluster.toml:ro - - ./secrets:/etc/big/secrets:ro - command: - - /data/big.db - - 0.0.0.0:7654 - - --cluster - - /etc/big/cluster.toml - - --node - - a-spare - - --users - # Staged from the mount by the entrypoint, which reads it as root and writes a private - # copy: a bind mount carries the host's ownership, which the image cannot predict. - - /run/big/users - # What this node presents to the other two. The CA that checks it is named in cluster.toml, + # What this node presents to the other node. The CA that checks it is named in cluster.toml, # because that half is the same everywhere; these two are this node's alone. - --peer-cert - /run/big/peer.pem @@ -182,6 +168,16 @@ services: # ceiling is a daemon one expensive query can hold a worker on for as long as it likes. - --query-timeout - "30" + # Trailing free pages go back to the filesystem while this node serves. Local, and gated + # on nothing but this switch: no peer is asked and nothing is proposed, so it does not + # care that this is a cluster. It waits for the file to reach 4096 pages with a quarter of + # it reusable, which a demo this small never reaches - it is here because this file is + # also a template. **What it gives back is the free run at the end of the file and nothing + # else**: a database that has churned in the middle keeps those holes until an offline + # `big compact` rewrites it, because moving a live page means rewriting whoever points at + # it. `big leaks ` prints the accounting, offline. Watch big_pages_reclaimed_total + # and big_reclaim_blocked_total; the second one climbing means readers, not a fault. + - --reclaim # **The same certificate again, and this is the pair that makes the cluster work.** # `--peer-cert` is the identity this node presents when it *dials* another; `--tls-cert` is # what its own listener presents when it is dialled. A node given only the first speaks TLS @@ -203,7 +199,7 @@ services: - --backup-dir - /data - # The front door. One address, three nodes behind it, and a request that lands on a node which + # The front door. One address, two nodes behind it, and a request that lands on a node which # has stopped serving lands somewhere else instead. # # It adds no routing: every node already plans a query and fans it out, so this answers "which @@ -219,7 +215,7 @@ services: # the clear. Add --tls-cert and --tls-key before publishing it wider. ports: - "127.0.0.1:7654:7650" - depends_on: [a, b, a-spare] + depends_on: [a, b] volumes: # The same file the nodes read. This takes `name` and `addr` out of it and ignores the # rest — it does not route by key, so the shard map is none of its business. One file, so @@ -255,4 +251,3 @@ services: volumes: a-data: b-data: - spare-data: diff --git a/deploy/cluster/users.sh b/deploy/cluster/users.sh index 3a510b2..5a33586 100755 --- a/deploy/cluster/users.sh +++ b/deploy/cluster/users.sh @@ -30,11 +30,15 @@ command -v "$big" >/dev/null || { exit 1 } -mkdir -p secrets -chmod 700 secrets +# Where to write it. `deploy/k8s` points this at its own directory, because a users file is a +# deployment's, not a machine's - see certs.sh for the same override and the same reason. +out=${SECRETS:-secrets} -if [ -f secrets/users ]; then - echo "secrets/users already exists; delete it first if you mean to rotate" >&2 +mkdir -p "$out" +chmod 700 "$out" + +if [ -f "$out/users" ]; then + echo "$out/users already exists; delete it first if you mean to rotate" >&2 exit 1 fi @@ -44,15 +48,15 @@ reader=$(random) # Piped rather than typed: `big passwd` reads one line from standard input when it is not a # terminal, which is the only non-tty path and exists for exactly this. It is never a flag, # because an argument is visible in `ps`. -printf '%s\n' "$admin" | "$big" passwd secrets/users set ops --role superuser +printf '%s\n' "$admin" | "$big" passwd "$out/users" set ops --role superuser # **Named before it exists**, and that is the right order: a role is resolved by name on every # request, so creating `dashboard` in the catalog later makes this credential work without a # restart. What needs a restart is changing the *name* in this file - it is read once, at # startup - which is the trap worth knowing about and the reason the name is chosen now. -printf '%s\n' "$reader" | "$big" passwd secrets/users set dashboard --role dashboard +printf '%s\n' "$reader" | "$big" passwd "$out/users" set dashboard --role dashboard echo -echo "wrote secrets/users (mode 600)" +echo "wrote $out/users (mode 600)" echo " ops: $admin (superuser)" echo " dashboard: $reader (role \`dashboard\`, which does not exist yet)" echo diff --git a/deploy/k8s/00-namespace.yaml b/deploy/k8s/00-namespace.yaml new file mode 100644 index 0000000..58c950e --- /dev/null +++ b/deploy/k8s/00-namespace.yaml @@ -0,0 +1,17 @@ +# One namespace, because every name in here is a DNS name the cluster depends on. +# +# `cluster.toml` says a node is reached at `a:7654`, and that short name resolves only from +# inside the namespace the Service `a` lives in - the resolver's search path is +# `.svc.cluster.local` first. Two copies of this deployment in one namespace would be +# two nodes called `a`; two namespaces is two clusters that cannot see each other, which is what +# you want and is free. +apiVersion: v1 +kind: Namespace +metadata: + name: big + labels: + # The restricted profile is what these manifests are written against: no root, no added + # capabilities, seccomp on. The entrypoint is happy unprivileged - see the readme, "Why + # nothing here runs as root and the Secret is mode 0440". + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest diff --git a/deploy/k8s/10-config.yaml b/deploy/k8s/10-config.yaml new file mode 100644 index 0000000..6feaf33 --- /dev/null +++ b/deploy/k8s/10-config.yaml @@ -0,0 +1,206 @@ +# Three lists, and they follow three different rules. Getting them confused is the whole +# difficulty of running this on Kubernetes, so they are in one file with the difference written +# down. +# +# **`seed.toml` is the membership, and it must be exact.** A node with no log seeds its agreement +# from this file and counts a majority out of it, so naming a node that is not running is naming +# a vote that never arrives: three entries with two pods up is a cluster that cannot elect +# anybody. It names the *founding* nodes and is not edited when the cluster grows - growth is a +# committed decision, and `bigctl cluster topology` is where the current membership is. +# +# **`upstreams.toml` is an address space, and it may name nodes that do not exist.** The proxy +# does not vote; a name it cannot reach is one it polls, marks unreachable and leaves out of +# rotation. So it is written out to the ceiling once, and a node that appears later is picked up +# within a health interval with no restart and no edit. Raise the ceiling here and in +# `secrets.sh` - those two lists are the same list. +# +# **`join.sh` writes the third one**, per pod, at startup: the file *that* node begins with. See +# the script. +apiVersion: v1 +kind: ConfigMap +metadata: + name: big-seed +data: + seed.toml: | + # The cluster as it is founded, and only that. `kubectl scale` does not edit this file; a + # node that joins afterwards is registered with `cluster add-node` and given a range by the + # balancer, and the agreement is what says who is in the cluster from then on. + # + # Three nodes and not two, because two is the size at which nothing works: a majority of two + # is two, so a cluster of two can neither fail a range over nor hand the schema-key + # namespace on - both are decisions a majority has to commit while one node is the one that + # went away. `deploy/cluster` runs two because compose runs on one machine and the second + # node is there to show the shape. This is not that. + cluster_id = "big-k8s" + + schema_leader = "big-0" + peer_ca_file = "/run/big/peer-ca.pem" + + # The names are the pods'; the addresses are the headless Service's. They differ, and the + # daemon is built for them differing: a certificate is issued to a *name*, and + # `ClientWire::connect` is handed `peer.name` as the TLS server name rather than the host it + # dialled. So a node keeps its identity wherever it is reached. + [[node]] + name = "big-0" + addr = "big-0.nodes:7654" + shards = "0..64" + + [[node]] + name = "big-1" + addr = "big-1.nodes:7654" + shards = "64..128" + + [[node]] + name = "big-2" + addr = "big-2.nodes:7654" + shards = "128.." + + upstreams.toml: | + # What the proxy may forward to. It reads `name` and `addr` and discards the rest - it does + # not route by key - so there are no ranges here and no ceiling on how wrong they could be. + # + # **Eight names for three nodes, on purpose.** A name whose pod does not exist is polled, + # found unreachable and left out of rotation; a name that appears is in rotation within a + # health interval. That is what makes `kubectl scale big --replicas=5` reach the client + # without a proxy restart. Nothing here grants anything: a certificate is what a node + # authenticates with, and the roster that accepts one follows the agreement. + cluster_id = "big-k8s" + + [[node]] + name = "big-0" + addr = "big-0.nodes:7654" + [[node]] + name = "big-1" + addr = "big-1.nodes:7654" + [[node]] + name = "big-2" + addr = "big-2.nodes:7654" + [[node]] + name = "big-3" + addr = "big-3.nodes:7654" + [[node]] + name = "big-4" + addr = "big-4.nodes:7654" + [[node]] + name = "big-5" + addr = "big-5.nodes:7654" + [[node]] + name = "big-6" + addr = "big-6.nodes:7654" + [[node]] + name = "big-7" + addr = "big-7.nodes:7654" +--- +# What each pod runs before its daemon starts: it writes the cluster file *that* node begins +# with, and registers the node with the cluster it is joining. +# +# **Both halves exist because of one rule in the daemon**: a node's listener accepts a peer whose +# certificate names somebody on its roster, and the roster it starts with comes from its own +# file. A node whose file names only itself refuses every peer that dials it, has none of its own +# to dial, and sits there healthy and alone. Once it is in touch the roster follows the +# agreement - `Controller::follow_roster` - so this is only about the first second. +# +# **It is a shell script and not an operator**, and the ceiling on what it will do is: write a +# file, and make one call. It does not choose ranges, does not admit anybody and does not scale +# anything. The balancer does those, on the leader, from facts it gathers itself. +apiVersion: v1 +kind: ConfigMap +metadata: + name: big-join +data: + join.sh: | + #!/bin/sh + # Runs as an initContainer. Never fails the pod: a cluster that cannot be reached yet is + # what a bootstrap looks like, and the daemon behind this is what makes it reachable. + set -eu + + me=${POD_NAME:?POD_NAME is not set} + svc=${PEER_SERVICE:?PEER_SERVICE is not set} + seed=/etc/big/seed/seed.toml + gen=/etc/big/gen/cluster.toml + ca=/etc/big/secrets/peer-ca.pem + cred=/etc/big/join/credentials + + # The `name = "..."` of every [[node]], in file order. Anchored past leading space so a + # commented-out entry is not read as one. + names() { sed -n 's/^[[:space:]]*name[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$1"; } + has() { names "$2" | grep -qx "$1"; } + + cp "$seed" "$gen" + + if has "$me" "$gen"; then + echo "join: $me is a founding node; starting from the seed unchanged" + else + # **Every name up to this pod's own ordinal, as a replica of the first.** + # + # Not just this node: the peer that dials a joining node is the agreement's *leader*, + # and by the time a cluster has grown the leader may be a node the seed never mentioned. + # A roster that named only the founders and this pod would refuse it and the join would + # hang for no visible reason. + # + # `replica` and not `shards` because it is the shape that parses without claiming a + # range anybody else owns - and a claim would be a lie this node might act on if it were + # ever alone. The agreement overwrites all of it within the second. Names of pods that + # do not exist, or that were removed, cost nothing: a roster is a list of names a + # certificate may claim, not a list of votes. + first=$(names "$gen" | head -1) + ord=${me##*-} + stem=${me%-*} + i=0 + while [ "$i" -le "$ord" ]; do + n="$stem-$i" + i=$((i + 1)) + if has "$n" "$gen"; then continue; fi + printf '\n[[node]]\nname = "%s"\naddr = "%s.%s:7654"\nreplica = "%s"\n' \ + "$n" "$n" "$svc" "$first" >> "$gen" + done + echo "join: wrote a cluster file naming $(names "$gen" | tr '\n' ' ')" + fi + + # A founding node is in the seed membership already; there is nobody to ask and nothing to + # add. + if has "$me" "$seed"; then + exit 0 + fi + + # **Registering is a separate act from being reachable, and it is meant to be.** A node that + # dials in is not thereby a member: `add-node` is a decision an existing cluster commits, + # and the credential below is what says somebody authorised it. Without one this pod still + # starts - it just sits outside the cluster until an operator runs the command printed here. + if [ ! -r "$cred" ]; then + echo "join: no credential mounted, so $me will not register itself. Run:" >&2 + echo "join: bigctl cluster add-node $me $me.$svc:7654" >&2 + exit 0 + fi + + # `bigctl` holds a credentials file to the same rule the daemon holds a users file to, and a + # Secret cannot be both readable here and mode 600 - see the readme. A private copy can. + install -m 600 "$cred" /run/join/credentials + + # **Before the daemon, not after**, which is the ordering that matters: a node that starts + # while the cluster has never heard of it is a voter in a cluster of its own imagining, and + # it will stand for election once per timeout until somebody tells it otherwise. Registered + # first, the leader reaches it as a learner - which does not vote and does not stand. + # + # Only the leader can commit this, and it does not forward, so this asks each node in turn. + for peer in $(names "$gen"); do + if [ "$peer" = "$me" ]; then continue; fi + if out=$(bigctl --addr "https://$peer.$svc:7654" --ca-file "$ca" \ + --credentials-file /run/join/credentials \ + cluster add-node "$me" "$me.$svc:7654" 2>&1); then + echo "join: $peer took the registration" + exit 0 + fi + # Already a member: a pod that restarted, or a second pass. Nothing to do and not a + # failure. + case $out in + *"already in this cluster"*) + echo "join: $me is already a member" + exit 0 + ;; + esac + done + + echo "join: nobody took the registration; starting anyway. The cluster may still be" >&2 + echo "join: bootstrapping - 'bigctl cluster topology' says where it stands." >&2 + exit 0 diff --git a/deploy/k8s/20-nodes.yaml b/deploy/k8s/20-nodes.yaml new file mode 100644 index 0000000..8fa4d34 --- /dev/null +++ b/deploy/k8s/20-nodes.yaml @@ -0,0 +1,295 @@ +# The nodes: one StatefulSet, and `kubectl scale` is how it grows. +# +# **What makes that possible is not this file.** A node that appears has to be given a +# certificate, a cluster file that names the cluster, a membership entry, and a range - four +# things, and only the first two are Kubernetes's. The daemon does the other two by itself: +# `add-node` registers the pod as a **learner**, which holds nothing and does not vote, and the +# balancer on the agreement's leader admits it once it has caught up and then cuts the tail of +# the shard space over to it - `Action::Admit` then `Action::SplitTail`, one decision per pass, +# each one committed. That is `--balance` below, and it is why there is no YAML to write per +# node. +# +# So the four become: a pre-issued certificate (`secrets.sh`, and names are cheap), a file the +# initContainer writes (`join.sh`), and two decisions the cluster makes about itself. +# +# **Scaling in is not the mirror image**, and deliberately so. Deleting a pod that holds a range +# takes that range offline, and `cluster remove` refuses a node that still holds one. Drain it +# first - see the readme. +apiVersion: v1 +kind: Service +metadata: + name: nodes + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node +spec: + # Headless, so `big-3.nodes` is the pod itself: peer traffic goes straight to the daemon, and + # a StatefulSet's pod gets the stable name its certificate was issued for. + clusterIP: None + # **The one Kubernetes-specific decision here.** Endpoints are the ready ones by default, so a + # node that has not passed its probe would have no address - and a peer that cannot reach it + # cannot help it settle the election that would make it ready. This is the flag that is not + # that deadlock, and it is what a joining pod is dialled at before it is anything. + publishNotReadyAddresses: true + selector: + app.kubernetes.io/name: big + app.kubernetes.io/component: node + ports: + - name: big + port: 7654 + targetPort: big +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: big + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node +spec: + serviceName: nodes + # **Three, and two would be worse than one.** A majority of two is two, so a two-node cluster + # can neither fail a range over nor hand the schema-key namespace on: both are decisions a + # majority has to commit at the moment one node is the one that went away. This is the + # smallest size at which the two flags below mean anything. + # + # `seed.toml` names exactly these three. Everything above three is a join, and joins are not + # in a file. + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node + # The default, written out because it looks like a trap and is not one: ordered start waits + # for each pod to be *ready*, and readiness here means "the listener answers" rather than "the + # range is being served". A bootstrap therefore does not deadlock waiting for a quorum that + # needs the next pod. See the readme on what a probe here can and cannot say. + podManagementPolicy: OrderedReady + # Deleting a StatefulSet must not be a way to delete a database. + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + template: + metadata: + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node + spec: + terminationGracePeriodSeconds: 2 + affinity: + podAntiAffinity: + # Preferred and not required: on a one-machine cluster - kind, minikube, what this is + # first run on - `required` would leave two of the three Pending forever. What it buys + # on a real cluster is that one machine failing is one range down rather than three. + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + # What makes the Secret's `0440` readable: its files are owned `root:`. It is + # also what makes the volume writable without the image chowning anything. + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + initContainers: + # Writes the cluster file this pod starts with, and registers the pod if it is new. + # Never fails the pod: a cluster that cannot be reached yet is what a bootstrap looks + # like. See the `big-join` ConfigMap for the whole of it. + - name: join + image: big:latest + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "/etc/big/join-script/join.sh"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: PEER_SERVICE + value: nodes + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 10m + memory: 32Mi + volumeMounts: + - name: gen + mountPath: /etc/big/gen + - name: seed + mountPath: /etc/big/seed + readOnly: true + - name: join-script + mountPath: /etc/big/join-script + readOnly: true + - name: secrets + mountPath: /etc/big/secrets + readOnly: true + - name: join-credential + mountPath: /etc/big/join + readOnly: true + - name: join-run + mountPath: /run/join + containers: + - name: big + image: big:latest + imagePullPolicy: IfNotPresent + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + args: + - /data/big.db + - 0.0.0.0:7654 + # The file the initContainer wrote, not the ConfigMap. A founding node's copy is the + # seed unchanged; a joining node's names the cluster it is joining as well. + - --cluster + - /etc/big/gen/cluster.toml + # Which entry in that file this daemon is. Kubernetes expands `$(POD_NAME)` from the + # environment above, so the pod's ordinal is the node's name and the certificate + # issued to it is the one the entrypoint stages - it reads `--node` out of this list + # rather than out of a second variable that could disagree with it. + - --node + - $(POD_NAME) + - --users + - /run/big/users + - --peer-cert + - /run/big/peer.pem + - --peer-key + - /run/big/peer.key + - --tls-cert + - /run/big/peer.pem + - --tls-key + - /run/big/peer.key + - --query-timeout + - "30" + # **On here, where the compose demo has it off.** This is the flag that makes + # `kubectl scale` a scale: on the agreement's leader it admits a learner that has + # caught up, gives a node with nothing the tail of the shard space - a split that + # moves no bytes - and moves a range off a node an operator is draining. One + # decision per pass, each committed, and it does nothing at all on a node that is + # not the leader. + # + # `deploy/cluster` leaves it off because two nodes and two ranges are the thing that + # file is explaining, and a cluster that redraws them mid-demo is explaining + # something else. Here the shape is meant to change. + - --balance + # Hands the row-key namespace on after fifteen seconds of silence from whoever holds + # it. Inert with two nodes and real with three: handing it on is a decision the + # agreement has to commit, and the moment it is needed is the moment its holder is + # the node that went away. + - --elect-schema-leader + # Purely local: it asks no peer and proposes nothing. Gives the free run at the end + # of the file back to the filesystem while this node serves. + - --reclaim + - --backup-dir + - /data + ports: + - name: big + containerPort: 7654 + # `scheme: HTTPS` with no certificate check, which is what is wanted: the kubelet + # dials the pod's IP and the certificate names `big-3`. The question is "can this + # process answer", not "is this the right process", and it never leaves the machine to + # ask it. The compose healthcheck passes `-k` for the same reason. + livenessProbe: + httpGet: { path: /health, port: big, scheme: HTTPS } + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + # **"Answering", not "serving", and it cannot be made to say more.** `/ready` is always + # `200` - `serving` is a field in the body - and an httpGet probe reads the status line + # only. The proxy is what reads the body. See 30-proxy.yaml. + readinessProbe: + httpGet: { path: /ready, port: big, scheme: HTTPS } + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + # A request, so the scheduler knows what it is placing. **No memory limit**: reads come + # out of the mapping, so most of what this process appears to use is page cache the + # kernel will give back, and a hard ceiling turns that into an OOM kill under load. + # Set one if you have measured your workload, not because a template had one. + resources: + requests: + cpu: 100m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + volumeMounts: + - name: data + mountPath: /data + # Where the entrypoint stages the credentials at mode 600. Must be writable, and + # must not outlive the pod: what is in it is a private copy of a private key. + - name: run + mountPath: /run/big + - name: gen + mountPath: /etc/big/gen + readOnly: true + - name: secrets + mountPath: /etc/big/secrets + readOnly: true + volumes: + - name: run + emptyDir: {} + - name: join-run + emptyDir: {} + # Written by the initContainer, read by the daemon. + - name: gen + emptyDir: {} + - name: seed + configMap: + name: big-seed + items: + - key: seed.toml + path: seed.toml + - name: join-script + configMap: + name: big-join + # **One Secret for every pod, and that is the price of one StatefulSet.** A pod stages + # only the certificate `--node` names, so no daemon presents anybody else's - but every + # pod could read every key, which per-node certificates were meant to avoid. Per-node + # Secrets need per-node workloads, which is the shape that cannot be scaled. If that + # trade is the wrong way round for you, issue per-pod certificates with cert-manager's + # CSI driver and mount that instead; nothing above changes. + # + # `0440` and not `0400`: the files are owned `root:` and the daemon is not + # root, so `0400` would be unreadable and the entrypoint would stage nothing - which + # presents as `cannot read /run/big/users`, not as a permission error on the mount. + # `0440` is also why the staging is not optional: `big serve` refuses a users file or a + # key that is group-readable, and no `defaultMode` is both readable here and mode 600. + - name: secrets + secret: + secretName: big-nodes + defaultMode: 0440 + # The credential the initContainer registers with, and the only optional volume here: a + # cluster with no `big-ops` Secret still starts, and prints the one `add-node` command an + # operator then runs by hand. + - name: join-credential + secret: + secretName: big-ops + defaultMode: 0440 + optional: true + volumeClaimTemplates: + - metadata: + name: data + spec: + # One volume per pod, never one shared: one process holds one file, and the pager takes + # an exclusive lock. + accessModes: [ReadWriteOnce] + # No `storageClassName`, so the cluster's default is used. Name one here if the default + # is network storage and this should be local - a database's latency is its disk's. + resources: + requests: + storage: 10Gi diff --git a/deploy/k8s/30-proxy.yaml b/deploy/k8s/30-proxy.yaml new file mode 100644 index 0000000..1879486 --- /dev/null +++ b/deploy/k8s/30-proxy.yaml @@ -0,0 +1,166 @@ +# The front door, and the only Service a client is given. +# +# It adds no routing - every node already plans a query and fans it out - so what it answers is +# "which node is alive", and that is the answer Kubernetes cannot produce here. `/ready` on a +# node is always `200`, with `serving` as a field in the body; an `httpGet` probe reads the +# status line and nothing else. This process reads the body, every couple of seconds, and takes +# a node that says `"serving":false` out of rotation without restarting anything. See +# crates/big-proxy/src/health.rs, which is written around exactly those two facts. +# +# It also adds a route allowlist: `/internal/*` is unreachable through it and cannot be made +# reachable, because it presents no client certificate either. +apiVersion: v1 +kind: Service +metadata: + name: big + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy +spec: + # **ClusterIP, and deliberately nothing else.** The compose file publishes to `127.0.0.1` + # because the hop below is in the clear; a ClusterIP is the same promise with the same + # boundary drawn around the cluster instead of the machine, and `kubectl port-forward` is the + # loopback hop. There is no Ingress and no LoadBalancer in this directory on purpose: adding + # one publishes a plaintext database port, and the fix is not an annotation but `--tls-cert` + # and `--tls-key` on this deployment and dropping `--insecure-no-tls` below. + type: ClusterIP + selector: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy + ports: + # 7654 outside, 7650 inside: a client that moves from the compose deployment to this one + # changes the host and nothing else. + - name: big + port: 7654 + targetPort: proxy +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: big-proxy + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy +spec: + # **Two, where compose has one.** A Deployment and not a StatefulSet because this is the one + # component here that genuinely is fungible: it opens no file, holds no lock, stores no + # credential and has no identity to keep. The compose file runs one of it because compose runs + # on one machine; a front door that is a single point of failure is the thing the front door + # was added to stop being. + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy + template: + metadata: + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy + spec: + terminationGracePeriodSeconds: 2 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: bigproxy + # A second image, and the separation is the point: this one has no engine, no SQL + # planner and no argon2 in it. See deploy/proxy/Dockerfile. + image: bigproxy:latest + imagePullPolicy: IfNotPresent + args: + - 0.0.0.0:7650 + - --cluster + - /etc/bigproxy/cluster/upstreams.toml + # Trusting the CA is not being trusted by it: this verifies the nodes' certificates + # and presents none of its own, which is why `/internal/*` stays out of reach. + - --upstream-ca + - /run/bigproxy/peer-ca.pem + # The hop from a client to here is in the clear, and the ClusterIP above is what + # bounds it. Give this `--tls-cert` and `--tls-key` and drop this flag before + # anything outside the cluster reaches it. + - --insecure-no-tls + ports: + - name: proxy + containerPort: 7650 + # Liveness: is this process answering. A proxy whose upstreams are all down is a + # healthy proxy - it is answering `503`, which is the correct answer - and restarting + # it would not bring a database back. + livenessProbe: + httpGet: { path: /health, port: proxy } + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + # Readiness: is there anywhere to send a request. `503` when nothing is in rotation, + # so this pod leaves the Service rather than accepting a request it can only refuse. + # It costs nothing to ask: the answer comes out of what the poller already knows, and + # the probe never touches a node. + readinessProbe: + httpGet: { path: /ready, port: proxy } + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 2 + resources: + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + volumeMounts: + - name: run + mountPath: /run/bigproxy + - name: cluster + mountPath: /etc/bigproxy/cluster + readOnly: true + - name: secrets + mountPath: /etc/bigproxy/secrets + readOnly: true + volumes: + - name: run + emptyDir: {} + # **Not the seed the nodes read**, and the difference is the point: this list names + # every address a node could ever have, including pods that do not exist yet. The proxy + # takes `name` and `addr` and ignores the rest - it does not route by key - so a name it + # cannot reach is simply out of rotation, and one that appears is in rotation within a + # health interval. That is what makes `kubectl scale` reach a client with no restart + # here. The seed cannot be written that way: a node counts a majority out of it. + - name: cluster + configMap: + name: big-seed + items: + - key: upstreams.toml + path: upstreams.toml + # **The CA and nothing else.** Not a users file: this process authenticates nobody, it + # forwards the credential the client sent to a node that does. Not a node key: a + # certificate signed by that CA would name a node and would reach `/internal/*`, and + # there is deliberately no flag here that loads one. + - name: secrets + secret: + secretName: big-proxy + defaultMode: 0440 +--- +# **The one budget worth writing, and the nodes deliberately have none.** Draining a machine may +# take one of these two and must not take both - that is a real constraint an eviction can +# respect, because a second proxy is as good as the first. +# +# A node's is not that shape. Each range lives on exactly one node here, so evicting either one +# takes a range offline no matter what a budget says; `maxUnavailable: 0` would not protect the +# data, it would hang `kubectl drain` forever and call it protection. What makes a node evictable +# is a copy of its range, and that needs the third node docs/clustering.md describes. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: big-proxy +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml new file mode 100644 index 0000000..0843cc6 --- /dev/null +++ b/deploy/k8s/kustomization.yaml @@ -0,0 +1,19 @@ +# `kubectl apply -k deploy/k8s` - after `./secrets.sh`, which is not listed here. +# +# **The Secrets are made by a script and not generated by kustomize**, on purpose. A +# `secretGenerator` would name them `big-node-a-` and roll the pods on every change, which +# for a credential file is usually the right instinct and here is not: rotating a node's +# certificate by restarting every node at once takes both ranges down, and the users file is +# read at startup anyway, so a rollout is a decision to make deliberately rather than a side +# effect of `apply`. It also keeps a private key out of every rendered manifest anybody pipes to +# a file. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: big + +resources: + - 00-namespace.yaml + - 10-config.yaml + - 20-nodes.yaml + - 30-proxy.yaml diff --git a/deploy/k8s/readme.md b/deploy/k8s/readme.md new file mode 100644 index 0000000..1ba6110 --- /dev/null +++ b/deploy/k8s/readme.md @@ -0,0 +1,277 @@ +# k8s + +Three nodes and a front door, and **`kubectl scale` is how the cluster grows**. The same binary +and the same flags as [`../cluster/`](../cluster/); what is different is that the shape is meant +to change here, so two flags that are off in the compose demo are on, and a pod that appears +gets itself into the cluster instead of waiting for somebody to write a file about it. + +## Running it + +There is no published image, so build both and put them where the cluster can see them: + +```sh +# From the repository root. +docker build -t big:latest . +docker build -t bigproxy:latest -f deploy/proxy/Dockerfile . + +kind load docker-image big:latest bigproxy:latest # or: minikube image load, or push and + # edit `image:` in the two manifests +``` + +Then the credentials and the manifests: + +```sh +cd deploy/k8s +./secrets.sh # a CA of its own, a certificate per pod name, and three Secrets. + # Prints three passwords - keep them. +kubectl apply -k . + +kubectl -n big rollout status statefulset/big +kubectl -n big port-forward svc/big 7654:7654 & +curl localhost:7654/ready +``` + +Two roles have to be made before the cluster is finished, and one of them is what lets it grow: + +```sh +ctl() { kubectl -n big exec -it big-0 -- \ + bigctl --addr https://big-0.nodes:7654 --ca-file /run/big/peer-ca.pem --user ops "$@"; } + +ctl sql "CREATE ROLE joiner" +ctl sql "GRANT OPERATE ON *.* TO joiner" # a new pod registers itself with this +ctl sql "CREATE ROLE dashboard" +ctl sql "GRANT OPERATE ON *.* TO dashboard" # /metrics, /verify +``` + +`bigctl` is in the image, the address is the node's *name* because that is what its certificate +says, and `--ca-file` is the CA the entrypoint staged inside the container. `--user` asks for the +password on the terminal, which is why that `exec` has `-it`; a script wants +`BIG_CREDENTIALS=` instead, one `user:password` line at mode 600, because a password given +as an argument is visible in `ps`. + +## Scaling out + +```sh +kubectl scale statefulset/big -n big --replicas=5 +ctl cluster topology +``` + +That is the whole of it, and what happens is worth following, because **only two of the four +steps are Kubernetes's**: + +1. **A certificate.** Already issued. `secrets.sh` mints one per *name* up to a ceiling, and a + name is cheap: a certificate grants nothing until the roster accepts it, and the roster + follows the agreement. So there is no `certs.sh` run and nothing to restart. +2. **A cluster file that names the cluster.** Written by the `join` initContainer, per pod, at + startup. A node's listener accepts a peer whose certificate names somebody on its roster, and + the roster it *starts* with comes from its own file - a node whose file named only itself + would refuse every peer that dialled it, have none of its own to dial, and sit there healthy + and alone. Once it is in touch the roster follows the agreement. +3. **A membership entry.** The same initContainer calls `cluster add-node`, before the daemon + starts. That registers the pod as a **learner**: it replicates the log, holds no range and + does not vote, so it neither answers for data it has not got nor raises the bar for an + election it could not help decide. +4. **A range.** Nobody asks for this. The balancer on the agreement's leader admits the learner + once it has caught up, and then cuts the tail of the shard space over to a node that holds + nothing - a split that moves no bytes. One decision per pass, each one committed. That is + `--balance`, and it is the flag `deploy/cluster` leaves off because two nodes and two ranges + are the thing that file is explaining. + +**The order in step 3 is not incidental.** A node that starts while the cluster has never heard +of it is a voter in a cluster of its own imagining, and it will stand for election once per +timeout - raising the term on every node that hears it - until somebody tells it otherwise. +Registered first, it is reached as a learner, and a learner does not stand. + +If the `joiner` role does not exist yet, or the `big-joiner` Secret is missing, the pod still +starts and its init container prints the one command to run: + +```sh +kubectl -n big logs big-4 -c join +# join: no credential mounted, so big-4 will not register itself. Run: +# join: bigctl cluster add-node big-4 big-4.nodes:7654 +``` + +**Past the ceiling.** `secrets.sh` issues eight names and the proxy is told about eight +addresses. `MAX=16 ./secrets.sh` issues the rest, and `upstreams.toml` in +[`10-config.yaml`](10-config.yaml) is the matching list - those two are the same list, and they +are the only two places a ceiling exists. + +## Scaling in is not the mirror image + +`kubectl scale --replicas=3` deletes `big-4`, and if `big-4` holds a range that range is +unanswerable until it comes back: nothing here is replicated, so the proxy picks a node that is +up and cannot pick data that is not there. **Take the work off it first:** + +```sh +ctl cluster drain big-4 # still votes, still coordinates; the balancer moves its ranges away +ctl cluster topology # wait until its `primary` column is empty +ctl cluster remove big-4 # refused while it still holds a range - that refusal is the point +kubectl scale statefulset/big -n big --replicas=4 +``` + +Draining is not removal, and removal is not deletion. Each is a separate decision because each +has a different failure: a node removed while it holds a range takes the only copy of it, and a +map that goes on naming a node nobody talks to is worse than one that says it is draining. + +The PVC is kept when a pod is scaled away (`whenScaled: Retain`), so scaling back up gives +`big-4` its file back rather than an empty one. That is what you want after an accidental scale +down and what you do not want after a real one - `kubectl delete pvc data-big-4` is the second +half of a scale-in that was meant. + +## The decisions + +### One StatefulSet, and `replicas` as the knob + +A node here is not interchangeable with the next one - it has a range, a certificate and a name - +which is the argument for one workload per node, and it is the wrong argument. What it produces +is a deployment where growing means writing YAML, and the cluster is already able to admit a +learner and give it work without being asked. So: one StatefulSet, pod names as node names, and +the parts that genuinely are per-node are handled where they belong - the certificate by a name +issued in advance, the cluster file by an initContainer, the range by the balancer. + +What ordinals cost is that the node names are Kubernetes's (`big-0`, not `a`), and that one +Secret is mounted by every pod. Both are written down where they bite: the second is on the +`secrets` volume in [`20-nodes.yaml`](20-nodes.yaml). + +### Three lists, three rules + +The mistake this deployment makes easy is treating them as one list. They are in one file, +[`10-config.yaml`](10-config.yaml), with the difference at the top: + +| | May name a node that does not exist | Why | +|---|---|---| +| `seed.toml` | **No** | A node with no log counts a majority out of it. Three entries with two pods up is a cluster that cannot elect anybody. | +| `upstreams.toml` | Yes | The proxy polls, finds it unreachable, leaves it out of rotation - and picks it up within a health interval when it appears. | +| the certificates | Yes | A certificate is checked against the roster, and the roster follows the agreement. An unused key grants nothing. | + +`seed.toml` names the founding three and is **not edited when the cluster grows**. Membership +after that is a committed decision, and `bigctl cluster topology` is where it is. A ConfigMap +edit does not reshape a running cluster and a rollout does not undo one. + +### Three nodes, not two + +Two is the size at which nothing works: a majority of two is two, so a two-node cluster can +neither fail a range over nor hand the schema-key namespace on - both are decisions a majority +has to commit at the moment one node is the one that went away. `deploy/cluster` runs two because +compose runs on one machine and the second node is there to show the shape. + +That is also why `--elect-schema-leader` is on here and off there: with two nodes it is inert +rather than merely cautious. + +### The peer Service publishes not-ready addresses + +A Service's endpoints are the ready ones by default. A node that has not passed its probe would +have no address; a peer that cannot reach it cannot help it settle the election that would make +it ready; both wait for each other on a cold start. `publishNotReadyAddresses: true` is the flag +that is not that deadlock, and it is also what a joining pod is dialled at before it is anything. + +### A probe can say "answering". It cannot say "serving" + +`GET /ready` on a node is **always `200`**. Whether it is actually serving its range is +`"serving": true|false` in the body, because a node that has lost touch with the agreement is +still perfectly healthy - it is one promotion away, and restarting it would help nothing. An +`httpGet` probe reads the status line and never the body. + +**That is what the proxy is for.** It polls `/ready` every couple of seconds, reads the field and +takes a node that says `false` out of rotation - see `crates/big-proxy/src/health.rs`, whose +module documentation is written around exactly this. So the Service a client is given is the +proxy's, and the proxy's own `/ready` *is* a real readiness signal: `503` when there is nowhere +to send a request, which takes that pod out of its Service. + +It also means `podManagementPolicy: OrderedReady` is safe here. If readiness meant "serving", an +ordered start would wait for a quorum that needs the next pod. + +### Why nothing runs as root, and the Secret is mode 0440 + +`big serve` refuses a users file or a private key that anyone but its owner can read. A Secret +volume cannot satisfy that directly: + +- its files are owned `root:`, and the daemon is uid 1000; +- `defaultMode: 0400` is therefore unreadable by the daemon - the entrypoint would silently stage + nothing, and the failure presents as `cannot read /run/big/users` rather than as a permission + error on the mount; +- `defaultMode: 0440` is readable, and is exactly what the check refuses. + +So the staging in [`../entrypoint.sh`](../entrypoint.sh) - written for bind mounts, whose +ownership the image cannot predict - is what makes a Secret work at all: it copies each file into +`/run/big` at mode 600, owned by whoever is running. `fsGroup: 1000` makes the source readable; +the private copy is what passes the check. The `join` initContainer does the same thing with its +credentials file, for the same reason. + +There is nothing left for root to do, so these pods start unprivileged: `runAsNonRoot`, no added +capabilities, `RuntimeDefault` seccomp, a read-only root filesystem with an `emptyDir` over +`/run/big`. The entrypoint notices it is not root and skips the privilege drop - the same path +`user:` in a compose file takes. The namespace enforces the `restricted` Pod Security profile, so +a manifest that regresses on any of that is rejected rather than quietly accepted. + +### ClusterIP, and no Ingress in this directory + +The compose file publishes to `127.0.0.1` because the hop from a client to the proxy is in the +clear. A ClusterIP is the same promise with the boundary drawn around the cluster instead of the +machine, and `kubectl port-forward` is the loopback hop. + +Adding an Ingress or a `type: LoadBalancer` here would publish a plaintext database port, so +neither is in this directory. The fix is not an annotation: give the proxy `--tls-cert` and +`--tls-key` from a Secret of its own and drop `--insecure-no-tls`, or terminate TLS in front of +it - [`../../runbook.md`](../../runbook.md) has a configuration that works. + +**`/verify`, `/repair` and `/admin/*` are not reachable through the proxy**, and that is the +allowlist rather than an omission: each asks about *one node*, and a proxy chooses which node +without telling you. `POST /admin/backup` is the sharpest case - through a front door it would +mean "back up a node, unspecified". `--allow-ops` turns them on if you want that anyway. + +## Rollouts, drains, and the range that has no copy + +No range here has a copy, so **stopping a node makes its range unanswerable until it comes +back**. Every operation below inherits that one fact. + +- **A rollout** deletes one pod at a time, so each node's range is down for as long as its pod + takes to come back. `kubectl rollout status` is the length of it. +- **`kubectl drain`** on a machine evicts the pod, which is the same thing. There is no + PodDisruptionBudget on the nodes on purpose: `maxUnavailable: 0` would not protect the data, it + would hang the drain forever and call that protection. What makes a node evictable is a copy of + its range. +- **The proxy has one**, `minAvailable: 1`, because a second proxy genuinely is as good as the + first - it holds no file, no lock and no identity. +- **A killed pod loses nothing.** A commit writes its pages, fsyncs, flips the meta page and + fsyncs again, so a process that dies leaves a file that is either before that flip or after it. + `terminationGracePeriodSeconds: 2` is generous. +- **The PVCs outlive the StatefulSet.** `kubectl delete -k .` leaves them behind and a re-apply + picks them back up: deleting a manifest must not be a way to delete a database. + +## Backups + +Per node, over the node's own port, and never with `big backup` - that subcommand wants the +exclusive lock the daemon is holding: + +```sh +kubectl -n big exec big-0 -- curl -sk -u ops:$PASSWORD -X POST \ + "https://127.0.0.1:7654/admin/backup?name=big-0-$(date +%F).db" +kubectl -n big cp big-0:/data/big-0-$(date +%F).db ./ +``` + +Each node holds its own range, so a copy of one node is a copy of one range, and the copies are +not one snapshot. Nothing here runs this for you: that is a CronJob and a bucket, and neither is +in this repository. + +## What is not here + +**An operator.** `bigctl cluster topology --format json` is the whole surface a placement +controller reads - which ranges exist, who holds each, what is moving - and `rebalance` is one +step against facts gathered afresh. What the initContainer does is the smallest thing that made +`kubectl scale` mean something: write a file, make one call. Everything past that - watching the +API, reacting to a node failure, deciding when to grow - is a controller's job and is not +pretended at here in eighty lines of shell. + +**A HorizontalPodAutoscaler.** Scaling out is cheap and scaling in is a sequence with a refusal +in the middle of it; a controller that scaled on CPU would eventually delete a pod holding the +only copy of a range. + +**Replication.** The balancer moves ranges; it does not make copies of them. A range with a copy +is what makes a rollout invisible and a node evictable, and setting one up is +[docs/clustering.md](../../docs/clustering.md). + +**Monitoring.** `/metrics` is Prometheus text on both, and the two are not the same route. The +proxy's is unauthenticated - it counts requests and latencies across a hop and says nothing about +anyone's data. A node's needs `OPERATE ON *.*`, is per node, and is not reachable through the +proxy, so a scrape config points at the `nodes` Service and carries the `dashboard` credential. diff --git a/deploy/k8s/secrets.sh b/deploy/k8s/secrets.sh new file mode 100755 index 0000000..216e2ea --- /dev/null +++ b/deploy/k8s/secrets.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# Everything this deployment needs on disk, and then in three Secrets. +# +# **A CA of its own, not `deploy/cluster`'s.** Two deployments sharing one CA is two deployments +# that can impersonate each other, and this one's node names are a StatefulSet's - `big-0`, +# `big-1` - rather than `a` and `b`. So it points the two scripts that already exist at a +# directory of its own; there is no cryptography here. +# +# ./secrets.sh # generate what is missing, then upload +# MAX=16 ./secrets.sh # room for sixteen pods rather than eight +# +# **Certificates are issued to the ceiling, not to the current size**, and that is safe in a way +# a membership list is not: a certificate names a node, and a node is let in by the *roster*, +# which follows the agreement rather than this directory. An unused key grants nothing. What it +# buys is that `kubectl scale big --replicas=6` needs no new certificate, no `certs.sh` run and +# no restart of anybody. +set -eu +cd "$(dirname "$0")" + +ns=${NS:-big} +out=${SECRETS:-secrets} +max=${MAX:-8} +big=${BIG:-big} + +command -v kubectl >/dev/null || { echo "no \`kubectl\` on PATH" >&2; exit 1; } + +# `big-0 big-1 ... big-`: the pod names a StatefulSet called `big` will hand out. +nodes="" +i=0 +while [ "$i" -lt "$max" ]; do + nodes="$nodes big-$i" + i=$((i + 1)) +done +nodes=${nodes# } + +# --------------------------------------------------------------------------------------------- +# On disk +# --------------------------------------------------------------------------------------------- + +if [ ! -f "$out/users" ]; then + command -v "$big" >/dev/null || { + echo "no \`$big\` on PATH; set BIG=/path/to/big. Hashing a password is argon2 and a" >&2 + echo "shell cannot do it - see ../cluster/users.sh." >&2 + exit 1 + } + SECRETS="../k8s/$out" ../cluster/users.sh + # **A third credential, and the reason it is not `ops`.** A pod that registers itself needs + # to reach `/admin/cluster/add-node`, which is `OPERATE ON *.*` - not superuser, and nothing + # else. Handing every pod the superuser password to save one `GRANT` would put full SQL + # access in a file every node can read. + # + # Named before the role exists, which is the right order and the same one `users.sh` uses + # for `dashboard`: a role is resolved by name on every request, so `CREATE ROLE joiner` + # after the cluster is up makes this credential work with no restart. Until then it + # authenticates and may do nothing - so joining does not work until an operator has decided + # it may. + joiner=$(od -An -tx1 -N24 /dev/urandom | tr -d ' \n') + printf '%s\n' "$joiner" | "$big" passwd "$out/users" set joiner --role joiner + printf 'joiner:%s\n' "$joiner" > "$out/join-credentials" + chmod 600 "$out/join-credentials" + echo + echo " joiner: $joiner (role \`joiner\`, which does not exist yet)" +fi + +if [ ! -f "$out/peer-ca.pem" ]; then + NODES="$nodes" SECRETS="../k8s/$out" ../cluster/certs.sh +else + # Already bootstrapped: issue only the names that are missing, against the CA that is there. + missing="" + for node in $nodes; do + [ -f "$out/$node.pem" ] || missing="$missing $node" + done + if [ -n "$missing" ]; then + # shellcheck disable=SC2086 + SECRETS="../k8s/$out" ../cluster/certs.sh $missing + fi +fi + +# --------------------------------------------------------------------------------------------- +# In the cluster +# --------------------------------------------------------------------------------------------- +# +# `create --dry-run=client | apply` rather than plain `create`: run twice, this rotates what is +# there instead of failing on the second run. A pod already running keeps what it staged at +# startup until it restarts - these files are read once. + +kubectl get namespace "$ns" >/dev/null 2>&1 || kubectl create namespace "$ns" + +# **One Secret for every pod**, because one StatefulSet is one PodSpec. Each pod stages only the +# certificate `--node` names, so no daemon presents anybody else's - but every pod could read +# every key. That is the trade one workload makes; 20-nodes.yaml says what to do if it is the +# wrong way round for you. +set -- --from-file=users="$out/users" --from-file=peer-ca.pem="$out/peer-ca.pem" +for node in $nodes; do + set -- "$@" --from-file="$node.pem=$out/$node.pem" --from-file="$node.key=$out/$node.key" +done +kubectl -n "$ns" create secret generic big-nodes "$@" \ + --dry-run=client -o yaml | kubectl -n "$ns" apply -f - + +# The CA alone. The proxy verifies node certificates and presents none - trusting a CA is not +# being trusted by it - and it authenticates nobody, so there is no users file here either. +kubectl -n "$ns" create secret generic big-proxy \ + --from-file=peer-ca.pem="$out/peer-ca.pem" \ + --dry-run=client -o yaml | kubectl -n "$ns" apply -f - + +# What a joining pod registers with. Optional in the manifest: without it a new pod still starts +# and prints the one `add-node` command an operator then runs by hand. +kubectl -n "$ns" create secret generic big-joiner \ + --from-file=credentials="$out/join-credentials" \ + --dry-run=client -o yaml | kubectl -n "$ns" apply -f - + +echo +echo "namespace $ns now holds:" +echo " big-nodes users, peer-ca.pem, and a certificate for each of: $nodes" +echo " big-proxy peer-ca.pem" +echo " big-joiner the credential a new pod registers itself with" +echo +echo "Once the cluster is up, make the two roles those credentials name:" +echo +echo " bigctl --user ops sql \"CREATE ROLE joiner\"" +echo " bigctl --user ops sql \"GRANT OPERATE ON *.* TO joiner\" # scaling out needs this" +echo " bigctl --user ops sql \"CREATE ROLE dashboard\"" +echo " bigctl --user ops sql \"GRANT OPERATE ON *.* TO dashboard\" # /metrics, /verify" +echo +echo "Until then \`kubectl scale\` starts a pod that cannot register itself, and says so in its" +echo "init container's log." diff --git a/deploy/readme.md b/deploy/readme.md index 2bdb412..bf735cb 100644 --- a/deploy/readme.md +++ b/deploy/readme.md @@ -1,15 +1,23 @@ # deploy -Two deployments, because they are two different things and pretending otherwise is how a +Three deployments, because they are three different things and pretending otherwise is how a single-node database ends up with a cluster's configuration and none of its guarantees. | | What it is | When | |---|---|---| | [`single/`](single/) | One node, every shard, no peers | Anything that fits on one machine | -| [`cluster/`](cluster/) | Two ranges and a copy of one | More data than one machine holds, or a range that has to survive losing one | +| [`cluster/`](cluster/) | Two ranges, one node each | More data than one machine holds | +| [`k8s/`](k8s/) | Three nodes that grow with `kubectl scale` | A cluster whose shape changes | + +**`cluster/` and `k8s/` are not the same deployment in two syntaxes.** The compose one is a fixed +shape being explained: two nodes, two ranges, and every flag that would redraw them turned off. +The Kubernetes one is a shape that changes: `--balance` is on, so a pod that appears is admitted +and given the tail of the shard space by the cluster's own leader, and a pod registers itself as +it starts. What that costs is a third node - a majority of two is two, so a two-node cluster can +commit neither a failover nor a schema-leader handover. **They are the same binary and the same code path.** `big serve` without `--cluster` builds itself a -cluster of one and runs every request through the same coordinator a three-machine deployment +cluster of one and runs every request through the same coordinator a multi-machine deployment does; a second path for the un-clustered case would be the path nobody tests. What the two directories differ in is a config file and how many containers there are. @@ -27,7 +35,7 @@ curl -u ops:change-me-please localhost:7654/ready names a role the catalog does not have is a credential that authenticates and may do nothing, which is what `--role admin` writes on a fresh database. -## Three nodes +## Two nodes ```sh cd cluster @@ -59,12 +67,16 @@ machine. The address is the node's *name* because that is what its certificate s on the terminal, which is what you want at one; a script wants `BIG_CREDENTIALS=` or `--credentials-file `, one `user:password` line at mode 600, because a password given as an argument is visible in `ps`. `serving: true` with a -`leader` is the line that says the three of them found each other; `serving: false` with +`leader` is the line that says the two of them found each other; `serving: false` with `leader: null` and a term that keeps climbing is the shape of a peer handshake that is failing. -Only `a` is published. Any node answers any request - the one that receives it plans the query -and fans it out - so publishing three ports would suggest a client has to choose, and it does -not. +Only the proxy is published. Any node answers any request - the one that receives it plans the +query and fans it out - so publishing a port per node would suggest a client has to choose, and +it does not. + +Neither range has a copy, so neither fails over: stop `a` and shards `0..64` are unanswerable +until it is back. That is the smallest shape that runs, not the one to run in production - "What +a joining node needs first" below adds the third node that makes a copy usable. ## Five things worth knowing before you run either @@ -157,7 +169,6 @@ ctl cluster topology # node addr state primary copy behind # a a:7654 voter 0..64 # b b:7654 voter 64..900 -# a-spare a-spare:7654 voter 0..64 # d d:7654 voter 900.. # bigctl: leader `a`, schema leader `a`, epoch 1 @@ -191,7 +202,7 @@ not help decide. `admit` is the step that makes it count. Three things, and the second is the one that is not obvious. **A certificate from the CA the cluster already trusts.** Re-running `./certs.sh` would mint a new -CA and lock out the three nodes that are talking, so it takes a name instead and signs one more +CA and lock out the nodes that are talking, so it takes a name instead and signs one more against the CA that is there: ```sh @@ -212,7 +223,7 @@ cluster_id = "big-demo" # the same string, or the cluster does not recogn schema_leader = "a" peer_ca_file = "/run/big/peer-ca.pem" -# ... a, b and a-spare exactly as they appear in cluster.toml ... +# ... a and b exactly as they appear in cluster.toml ... [[node]] name = "d" @@ -266,7 +277,7 @@ is a command-line decision rather than a request one because a request that chos could write anywhere the process can. In a cluster this is **per node**: each holds its own range, and a copy of one node is a copy of -one range - and the copies are not one snapshot. Only `a` is published, so the other two are +one range - and the copies are not one snapshot. Only the proxy is published, so the nodes are reached from inside, over their own TLS: ```sh diff --git a/docs/clustering.md b/docs/clustering.md index aa64465..f45e557 100644 --- a/docs/clustering.md +++ b/docs/clustering.md @@ -344,6 +344,74 @@ later. This is the same shape as every other refusal in the engine — see the more here than anywhere, because the alternative is two row ids for one string, which is exactly the failure per-node ranges were rejected for. +**A node that has just started has heard from nobody, and says so.** The lease is "when did I +last prove I was in touch with the agreement", and a fresh process has never proved it — so a +node holding a replicated range refuses reads and writes for its range, and a node named as the +schema leader refuses to assign row ids, until the first heartbeat reaches it. On the shipped +clocks that is one election. It is not politeness: a node that restarted may have been replaced +while it was away, and serving on the strength of a clock that also starts at zero is exactly +the two-primaries failure the lease exists to prevent. A cluster with no replicas is unaffected +— nothing there can be taken away, so nothing is fenced. + +**And the leader checks that it still is one.** Being named by the map is not the same as +being allowed to act on it, and a request to intern or allocate is refused, `503`, in three +cases. `not_schema_leader`: the map names somebody else — a coordinator holding a map one +decision old sent this to whoever led *then*, and the answer names who leads now. The request +carries who the sender *thinks* leads, so a sender that is behind is told rather than served. +`not_schema_leader` again: this node was told to stand down by a move that has not landed yet. +And `schema_lease_lost`: this node has lost touch with the agreement and cannot know it has not +been replaced — the same lease a replicated range serves under, and here there is no unfenced +case, because the namespace can always be given to somebody else. What these cost is one thing: +a write with a new key waits. What they prevent is two nodes interning at once, which no write +could wait out. + +**Moving the namespace has a step in the middle where nobody holds it.** `bigctl cluster +schema-leader --to ` copies every row key, then tells the old leader to stop, then reads +its allocation floor — final now, rather than racing — then commits. Between "stop" and the +commit, a write with a new key is refused; that gap is allowed. The gap it replaces, in which +the old leader went on interning for coordinators that had not yet heard, is not. + +**And with `--elect-schema-leader`, the agreement moves it on its own.** Off by default, and +the default is the honest one: a dead schema leader means no *new* row key until it is back, +which is an outage an operator can wait out, and the alternative has a cost that is worth +stating before it is switched on. + +It waits far longer than a range does — fifteen seconds against `promote_after`'s four and a +half — because moving the namespace copies every row key of every table, and a flap would be +the most expensive one available. The successor is chosen the way a range's replacement is: a +voter that is answering and not marked behind, in a fixed order, so two leaders elected in +sequence choose the same node. The decision sets `schema_ready` false, and until the successor +holds every key the survivors have, **nobody interns** — a write with a new key is refused +`schema_handover`, `503`. That gap is the same shape as the manual move's, and it is allowed +for the same reason. + +**What it costs, stated rather than hidden.** The successor's mapping is rebuilt from the +copies every *owner* took when it was told what a key meant — the dead leader is not there to +ask. A key it interned for a write that then failed exists nowhere else, so the successor will +hand that row id to a different string. Inside the surviving cluster that is harmless: nobody +holds the old meaning. It matters only if the deposed node comes back, and there it fails +loudly rather than quietly — `KeyStore::assign` refuses the contradiction, so `POST /repair` +reports it instead of overwriting. That is why the deposed node is marked behind whether or not +it held a copy of anything: **a node the agreement has taken the namespace from must be +repaired before it is trusted again.** Closing this properly means replicating a key assignment +before `intern` returns, which is a replicated catalog and a different design. + +If two *surviving* nodes disagree about what a row id means, the handover stops rather than +picking a side: `schema_ready` stays false, `big_cluster_schema_handover_blocked` goes to 1, +and nothing assigns row ids until somebody looks. A cluster that refuses new keys is +recoverable; a cluster with two meanings for one row id is not. + +**Record ids are reserved through the agreement before they are handed out.** The leader +allocates from a number in its memory, and a successor that started from what is on disk would +re-issue ids the old leader had promised to writes that had not landed — the one failure with no +second chance. So the map carries a ceiling per table, the leader commits a block of 65 536 ids +before it hands out anything under it, and a leader with no floor of its own — just elected, or +just restarted — starts at the ceiling. What a dead leader promised is below it by construction. +What it costs is one entry in the agreement per block, and a gap of unused ids when a leader is +replaced mid-block; record ids are sixty-four bits wide and shard-mapped, so a gap costs nothing. +This is still not consensus over the write path: the ids themselves travel as they always did, +and the agreement records only how far the leader may go. + ## The routes a node uses to reach another Thirteen, all `POST`, all under `/internal/`, all with a binary body and a binary answer. They are diff --git a/docs/proxy.md b/docs/proxy.md index 070c6a8..50f8ef3 100644 --- a/docs/proxy.md +++ b/docs/proxy.md @@ -255,12 +255,24 @@ cd deploy/cluster ./users.sh ; ./certs.sh docker compose up -d curl -u ops:$PASSWORD localhost:7654/ready +``` + +**That deployment holds no copy of either range, so it cannot show the thing this page is +about.** `docker compose stop a` takes `a` out of rotation and `in_rotation` drops, exactly as +it should - but shards `0..64` are then held by nobody, and a query reaching them fails at `b` +rather than at the front door. The proxy removed one single point of failure; an unreplicated +range is the other one, and it is not a proxy's to remove. + +The demonstration that matters needs a range with a copy. Add a third node as `replica = "a"` - +`deploy/readme.md`, "What a joining node needs first" - and then: + +``` docker compose stop a curl -u ops:$PASSWORD localhost:7654/ready # in_rotation drops, queries keep working ``` -That last pair is the only demonstration that matters: stop the machine a client was reaching, -and the client keeps working. +Stop the machine a client was reaching, and the client keeps working. That is the pair worth +running, and it takes both halves: a copy of the range, and one address in front of both. ## What is deliberately absent diff --git a/runbook.md b/runbook.md index a42d6b9..e588f66 100644 --- a/runbook.md +++ b/runbook.md @@ -173,6 +173,14 @@ happen online is the *swap*: `read` hands back a page borrowed straight out of t replacing the file under a live mapping is exactly the dangling reference the pager's four mmap constraints exist to prevent. So the copy runs while serving and only the rename needs the stop. +- **`--reclaim` gives the tail back without stopping anything.** A node started with it hands + trailing free pages to the filesystem while it serves, once a quarter of the file is + reclaimable and it finds a moment with no query in flight. It is not a substitute for + `compact`: only pages *flush against the end* are released, so one live page near the top + pins every free page below it. A file that churns comes down; a file that was emptied and + then left alone stays where it is. `big_pages_reclaimed_total` and + `big_reclaim_blocked_total` are the two numbers - the second climbing while the first stays + flat is a node that is never idle long enough. - **`compact` itself is offline.** It takes the exclusive lock, so stop `big serve` first. Running it against a served database fails with a lock error rather than doing anything behind the daemon's back. - Needs free space for a second copy of the *live data* alongside the original, briefly. @@ -751,7 +759,7 @@ in the group already holds it. ### Containers Two compose files, in [deploy/](deploy/): [`single/`](deploy/single/) for one node and -[`cluster/`](deploy/cluster/) for three. Same binary, same code path - `big serve` without +[`cluster/`](deploy/cluster/) for two. Same binary, same code path - `big serve` without `--cluster` builds itself a cluster of one - so what differs is a config file and how many containers there are. @@ -889,6 +897,29 @@ rate(big_storage_writes_total[15m]) / rate(big_txn_id[15m]) > 100 # Configured for full durability but not flushing. The two disagreeing is a bug, not a setting. big_durability{level="none"} == 0 and rate(big_storage_syncs_total[15m]) == 0 and rate(big_txn_id[15m]) > 0 + +# A copy is behind. Redundancy the cluster has lost, and nothing gets it back but POST /repair. +big_cluster_copies_behind > 0 + +# A range has been moving for longer than a move takes. The balancer plans nothing, cluster-wide, +# while anything is marked moving, and nothing clears the mark on its own: `bigctl cluster cancel`. +big_cluster_ranges_moving > 0 for 10m + +# A peer keeps not saying what it weighs. To the balancer it is neither a source nor a +# destination, so a cluster with one silent member quietly stops reshaping. +rate(big_cluster_peer_load_unanswered_total[15m]) > 0 + +# A move landed but its source could not let go of its copy. Space nobody reclaims until somebody +# looks, and the loop that moved it will not say so twice. +increase(big_cluster_balance_drops_failed_total[1h]) > 0 + +# Nobody holds the row-key namespace. A write with a key nobody has seen is refused while this +# is 0, so a handover that does not close in a minute is one that has stalled. +big_cluster_schema_ready == 0 for 1m + +# **Page somebody.** Handing the namespace over found two surviving nodes disagreeing about what +# a row id means. Nothing assigns row ids until this is resolved, and nothing clears it by itself. +big_cluster_schema_handover_blocked > 0 ``` Do **not** alert on `4xx`. It is clients being wrong, which is normal traffic.