From 9802f7d63b8eb58865ee4c24423c234834ab37b0 Mon Sep 17 00:00:00 2001 From: 9bany Date: Sat, 5 Sep 2026 00:52:21 +0700 Subject: [PATCH 1/6] chore(deploy): drop the a-spare replica from the cluster demo The demo cluster is two nodes now, `a` over `0..64` and `b` over `64..`, with no copy of either range. `config.rs:486` only requires three nodes once a `replica` is declared, so the two-node file is valid. What goes with it is every claim that is no longer true. The compose header promised `stop a` and the queries keep working; docs/proxy.md used `docker compose stop a` as the demonstration the proxy exists for, and called it "the only demonstration that matters". Neither holds without a copy: the proxy takes the node out of rotation exactly as designed, and shards 0..64 are then held by nobody, so the query fails at `b` rather than at the front door. The proxy removed one single point of failure; an unreplicated range is the other one, and it is not a proxy's to remove. docs/proxy.md now shows what adding a third node as `replica = "a"` buys and points at deploy/readme.md for it. Two pre-existing staleness fixes ride along, both in text the change touches: "Only `a` is published" in two places, which stopped being true when the proxy took the port. --- crates/big-proxy/tests/config.rs | 2 +- deploy/cluster/certs.sh | 4 +- deploy/cluster/cluster.toml | 16 +++---- deploy/cluster/docker-compose.yml | 70 ++++++------------------------- deploy/readme.md | 25 ++++++----- docs/proxy.md | 16 ++++++- runbook.md | 2 +- 7 files changed, 49 insertions(+), 86 deletions(-) diff --git a/crates/big-proxy/tests/config.rs b/crates/big-proxy/tests/config.rs index 3d88458..25fe4f8 100644 --- a/crates/big-proxy/tests/config.rs +++ b/crates/big-proxy/tests/config.rs @@ -39,7 +39,7 @@ fn both_readers_agree_on_the_shipped_cluster_file() { let proxy_pairs: Vec<(String, String)> = proxy.into_iter().map(|n| (n.name, n.addr)).collect(); assert_eq!(daemon_pairs, proxy_pairs, "the readers disagree about who is in the cluster"); - assert_eq!(daemon_pairs.len(), 3, "the shipped example is three nodes"); + assert_eq!(daemon_pairs.len(), 2, "the shipped example is two nodes, a range each"); } /// The divergence, pinned. diff --git a/deploy/cluster/certs.sh b/deploy/cluster/certs.sh index 66364a6..c837fbf 100755 --- a/deploy/cluster/certs.sh +++ b/deploy/cluster/certs.sh @@ -11,7 +11,7 @@ set -eu cd "$(dirname "$0")" -nodes="a b a-spare" +nodes="a b" days=${DAYS:-365} mkdir -p secrets @@ -20,7 +20,7 @@ cd secrets # **With a name, this issues one more certificate and leaves everything else alone.** A node that # joins a running cluster needs one, and the alternative - re-running the whole script - would -# rotate the CA and lock out the three nodes that are already talking. Called with no arguments it +# rotate the CA and lock out the nodes that are already talking. Called with no arguments it # is the bootstrap it always was, and refuses to run twice for the same reason. # # ./certs.sh the CA and a certificate for each of the nodes above diff --git a/deploy/cluster/cluster.toml b/deploy/cluster/cluster.toml index db102ea..2523c99 100644 --- a/deploy/cluster/cluster.toml +++ b/deploy/cluster/cluster.toml @@ -1,8 +1,9 @@ -# Three nodes: two ranges and a copy of one of them. +# Two nodes: a range each, and no copy of either. # -# Three is the minimum once anything has a copy, and `big serve` refuses fewer: failing over is a -# decision a majority has to agree on, and a majority of two is two. `b` counts towards that -# majority even though it holds a different range. +# Nothing here is replicated, so nothing fails over - a node that stops takes its range with it +# until it comes back. A copy would need a third node and not a second: failing over is a +# decision a majority has to agree on, and a majority of two is two, so two machines can never +# use a copy they hold. See docs/clustering.md, "Replication, and failing over". # # The addresses are the compose service names. Inside the network each node listens on # 0.0.0.0:7654 and reaches the others by name, which is why each one is started with --node: @@ -44,10 +45,3 @@ shards = "0..64" name = "b" addr = "b:7654" shards = "64.." - -# A copy of everything `a` holds. No range of its own: it takes `a`'s. If the node serving that -# range stops answering, the agreement gives the range to this one. -[[node]] -name = "a-spare" -addr = "a-spare:7654" -replica = "a" diff --git a/deploy/cluster/docker-compose.yml b/deploy/cluster/docker-compose.yml index f57dfb6..a031114 100644 --- a/deploy/cluster/docker-compose.yml +++ b/deploy/cluster/docker-compose.yml @@ -1,5 +1,5 @@ -# Three nodes, one of them a copy. See cluster.toml for the shape and docs/clustering.md for -# why it is that shape. +# Two nodes, a range each. See cluster.toml for the shape and docs/clustering.md for why it is +# that shape. # # cd deploy/cluster # ./users.sh # writes secrets/users, mode 600, and prints the passwords @@ -19,9 +19,13 @@ # So the published port is `proxy` now. A client still sees one address; it is no longer one # machine. # -# `stop a` and the queries keep working. That is the whole difference. +# **Nothing here is replicated, and that is a choice with a price.** Each range lives on exactly +# one node, so `stop b` leaves `64..` unanswerable until it comes back: the proxy picks a node +# that is up, it cannot pick data that is not there. Failing over needs a copy of a range, and a +# copy needs a third node to settle the vote - see docs/clustering.md, "Replication, and failing +# over", and deploy/readme.md, "What a joining node needs first", for what adds one. -# Everything the three nodes share. Not the volumes: a merge key replaces a list rather than +# Everything the two nodes share. Not the volumes: a merge key replaces a list rather than # extending it, so a `volumes` here would be silently overridden by each service's own and read # as though it applied. x-node: &node @@ -73,7 +77,7 @@ services: # Staged from the mount by the entrypoint, which reads it as root and writes a private # copy: a bind mount carries the host's ownership, which the image cannot predict. - /run/big/users - # What this node presents to the other two. The CA that checks it is named in cluster.toml, + # What this node presents to the other node. The CA that checks it is named in cluster.toml, # because that half is the same everywhere; these two are this node's alone. - --peer-cert - /run/big/peer.pem @@ -122,7 +126,7 @@ services: # Staged from the mount by the entrypoint, which reads it as root and writes a private # copy: a bind mount carries the host's ownership, which the image cannot predict. - /run/big/users - # What this node presents to the other two. The CA that checks it is named in cluster.toml, + # What this node presents to the other node. The CA that checks it is named in cluster.toml, # because that half is the same everywhere; these two are this node's alone. - --peer-cert - /run/big/peer.pem @@ -154,56 +158,7 @@ services: - --backup-dir - /data - a-spare: - <<: *node - volumes: - - spare-data:/data - - ./cluster.toml:/etc/big/cluster.toml:ro - - ./secrets:/etc/big/secrets:ro - command: - - /data/big.db - - 0.0.0.0:7654 - - --cluster - - /etc/big/cluster.toml - - --node - - a-spare - - --users - # Staged from the mount by the entrypoint, which reads it as root and writes a private - # copy: a bind mount carries the host's ownership, which the image cannot predict. - - /run/big/users - # What this node presents to the other two. The CA that checks it is named in cluster.toml, - # because that half is the same everywhere; these two are this node's alone. - - --peer-cert - - /run/big/peer.pem - - --peer-key - - /run/big/peer.key - # **Set it on a new deployment.** There is no default, on purpose - a query that used to - # finish must not start failing because the server was upgraded - but a daemon with no - # ceiling is a daemon one expensive query can hold a worker on for as long as it likes. - - --query-timeout - - "30" - # **The same certificate again, and this is the pair that makes the cluster work.** - # `--peer-cert` is the identity this node presents when it *dials* another; `--tls-cert` is - # what its own listener presents when it is dialled. A node given only the first speaks TLS - # outward into a plaintext port and every peer handshake fails - `received corrupt message - # of type InvalidContentType`, an election that never settles, `serving: false`. One - # listener serves peers and people both, so there is one certificate for both, and - # `certs.sh` issues it with `serverAuth` and `clientAuth` for exactly this reason. - # - # It also means the published port is HTTPS: `bigctl --addr https://... --ca-file` rather - # than a bare host:port. Client certificates are optional on this listener, so a person - # with no certificate still connects and still authenticates with a password. - - --tls-cert - - /run/big/peer.pem - - --tls-key - - /run/big/peer.key - # Per node, because a backup is per node: each holds its own range, and a copy of one node - # is a copy of one range. `big backup` cannot do this while the daemon runs - it wants the - # exclusive lock the daemon holds - so `POST /admin/backup` is the online path. - - --backup-dir - - /data - - # The front door. One address, three nodes behind it, and a request that lands on a node which + # The front door. One address, two nodes behind it, and a request that lands on a node which # has stopped serving lands somewhere else instead. # # It adds no routing: every node already plans a query and fans it out, so this answers "which @@ -219,7 +174,7 @@ services: # the clear. Add --tls-cert and --tls-key before publishing it wider. ports: - "127.0.0.1:7654:7650" - depends_on: [a, b, a-spare] + depends_on: [a, b] volumes: # The same file the nodes read. This takes `name` and `addr` out of it and ignores the # rest — it does not route by key, so the shard map is none of its business. One file, so @@ -255,4 +210,3 @@ services: volumes: a-data: b-data: - spare-data: diff --git a/deploy/readme.md b/deploy/readme.md index 2bdb412..3747fc3 100644 --- a/deploy/readme.md +++ b/deploy/readme.md @@ -6,10 +6,10 @@ single-node database ends up with a cluster's configuration and none of its guar | | What it is | When | |---|---|---| | [`single/`](single/) | One node, every shard, no peers | Anything that fits on one machine | -| [`cluster/`](cluster/) | Two ranges and a copy of one | More data than one machine holds, or a range that has to survive losing one | +| [`cluster/`](cluster/) | Two ranges, one node each | More data than one machine holds | **They are the same binary and the same code path.** `big serve` without `--cluster` builds itself a -cluster of one and runs every request through the same coordinator a three-machine deployment +cluster of one and runs every request through the same coordinator a multi-machine deployment does; a second path for the un-clustered case would be the path nobody tests. What the two directories differ in is a config file and how many containers there are. @@ -27,7 +27,7 @@ curl -u ops:change-me-please localhost:7654/ready names a role the catalog does not have is a credential that authenticates and may do nothing, which is what `--role admin` writes on a fresh database. -## Three nodes +## Two nodes ```sh cd cluster @@ -59,12 +59,16 @@ machine. The address is the node's *name* because that is what its certificate s on the terminal, which is what you want at one; a script wants `BIG_CREDENTIALS=` or `--credentials-file `, one `user:password` line at mode 600, because a password given as an argument is visible in `ps`. `serving: true` with a -`leader` is the line that says the three of them found each other; `serving: false` with +`leader` is the line that says the two of them found each other; `serving: false` with `leader: null` and a term that keeps climbing is the shape of a peer handshake that is failing. -Only `a` is published. Any node answers any request - the one that receives it plans the query -and fans it out - so publishing three ports would suggest a client has to choose, and it does -not. +Only the proxy is published. Any node answers any request - the one that receives it plans the +query and fans it out - so publishing a port per node would suggest a client has to choose, and +it does not. + +Neither range has a copy, so neither fails over: stop `a` and shards `0..64` are unanswerable +until it is back. That is the smallest shape that runs, not the one to run in production - "What +a joining node needs first" below adds the third node that makes a copy usable. ## Five things worth knowing before you run either @@ -157,7 +161,6 @@ ctl cluster topology # node addr state primary copy behind # a a:7654 voter 0..64 # b b:7654 voter 64..900 -# a-spare a-spare:7654 voter 0..64 # d d:7654 voter 900.. # bigctl: leader `a`, schema leader `a`, epoch 1 @@ -191,7 +194,7 @@ not help decide. `admit` is the step that makes it count. Three things, and the second is the one that is not obvious. **A certificate from the CA the cluster already trusts.** Re-running `./certs.sh` would mint a new -CA and lock out the three nodes that are talking, so it takes a name instead and signs one more +CA and lock out the nodes that are talking, so it takes a name instead and signs one more against the CA that is there: ```sh @@ -212,7 +215,7 @@ cluster_id = "big-demo" # the same string, or the cluster does not recogn schema_leader = "a" peer_ca_file = "/run/big/peer-ca.pem" -# ... a, b and a-spare exactly as they appear in cluster.toml ... +# ... a and b exactly as they appear in cluster.toml ... [[node]] name = "d" @@ -266,7 +269,7 @@ is a command-line decision rather than a request one because a request that chos could write anywhere the process can. In a cluster this is **per node**: each holds its own range, and a copy of one node is a copy of -one range - and the copies are not one snapshot. Only `a` is published, so the other two are +one range - and the copies are not one snapshot. Only the proxy is published, so the nodes are reached from inside, over their own TLS: ```sh diff --git a/docs/proxy.md b/docs/proxy.md index 070c6a8..50f8ef3 100644 --- a/docs/proxy.md +++ b/docs/proxy.md @@ -255,12 +255,24 @@ cd deploy/cluster ./users.sh ; ./certs.sh docker compose up -d curl -u ops:$PASSWORD localhost:7654/ready +``` + +**That deployment holds no copy of either range, so it cannot show the thing this page is +about.** `docker compose stop a` takes `a` out of rotation and `in_rotation` drops, exactly as +it should - but shards `0..64` are then held by nobody, and a query reaching them fails at `b` +rather than at the front door. The proxy removed one single point of failure; an unreplicated +range is the other one, and it is not a proxy's to remove. + +The demonstration that matters needs a range with a copy. Add a third node as `replica = "a"` - +`deploy/readme.md`, "What a joining node needs first" - and then: + +``` docker compose stop a curl -u ops:$PASSWORD localhost:7654/ready # in_rotation drops, queries keep working ``` -That last pair is the only demonstration that matters: stop the machine a client was reaching, -and the client keeps working. +Stop the machine a client was reaching, and the client keeps working. That is the pair worth +running, and it takes both halves: a copy of the range, and one address in front of both. ## What is deliberately absent diff --git a/runbook.md b/runbook.md index a42d6b9..f9ae047 100644 --- a/runbook.md +++ b/runbook.md @@ -751,7 +751,7 @@ in the group already holds it. ### Containers Two compose files, in [deploy/](deploy/): [`single/`](deploy/single/) for one node and -[`cluster/`](deploy/cluster/) for three. Same binary, same code path - `big serve` without +[`cluster/`](deploy/cluster/) for two. Same binary, same code path - `big serve` without `--cluster` builds itself a cluster of one - so what differs is a config file and how many containers there are. From 8f1c49c37f02bd50faec85d34cd835f40a67b966 Mon Sep 17 00:00:00 2001 From: 9bany Date: Sat, 5 Sep 2026 00:52:47 +0700 Subject: [PATCH 2/6] perf(pager): allocate the lowest reusable page, so the tail can drain The freelist was ordered by generation, so `alloc` handed out the oldest free page before the lowest one. Correct, and hopeless for the tail: writes kept landing at the end of the file while holes near the front stayed holes, so nothing was ever flush against EOF for `truncate_tail` to release and a file that had churned only ever grew. Ordering the runs by page number instead makes ordinary writes fill from the front. Nothing about reclaim changes: `alloc` still skips any run newer than the horizon, so this decides *which* reusable page is taken and never *whether* a page is reusable. The merge is unaffected - two runs that meet in page order can have nothing between them - and so is the termination argument for `alloc_freelist_pages`, which rests on `alloc` taking from the front of a run rather than on the order of runs. The rule is pinned against `Freelist` directly rather than through a commit, for the reason the module header of `reclamation.rs` already gives: a commit takes pages for its own chains before the caller's `alloc` sees the list, so a page number observed through one says as much about the machinery as about the rule. --- crates/big-pager/src/freelist.rs | 21 +++++++- crates/big-pager/tests/reclamation.rs | 71 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/crates/big-pager/src/freelist.rs b/crates/big-pager/src/freelist.rs index 75f6412..924cca5 100644 --- a/crates/big-pager/src/freelist.rs +++ b/crates/big-pager/src/freelist.rs @@ -86,11 +86,24 @@ impl Freelist { } /// Merges adjacent runs sharing a `freed_at`. This is where the run-length encoding pays off. + /// + /// **Ordered by page number, and that is what makes the file able to shrink.** It used to + /// be ordered by `freed_at` first, so [`Freelist::alloc`] handed out the oldest generation + /// before the lowest page - which is fine for correctness and hopeless for the tail: holes + /// near the front of the file stayed holes while writes kept landing at the end, and + /// nothing was ever flush against EOF for [`Freelist::trim_tail`] to release. Lowest first, + /// and ordinary writes fill from the front until the tail is free. + /// + /// The reclaim rule is untouched: `alloc` still skips any run newer than the horizon, so + /// this changes *which* reusable page is chosen and never whether one is reusable. The + /// merge is unaffected too - two runs that meet in page order cannot have another between + /// them - and so is the termination argument in `alloc_freelist_pages`, which rests on + /// `alloc` taking from the front of a *run* rather than on the order of the runs. pub fn compact(&mut self) { if self.runs.len() < 2 { return; } - self.runs.sort_unstable(); + self.runs.sort_unstable_by_key(|r| (r.first, r.freed_at)); let mut merged: Vec = Vec::with_capacity(self.runs.len()); for r in self.runs.drain(..) { match merged.last_mut() { @@ -106,7 +119,11 @@ impl Freelist { self.runs = merged; } - /// Takes one reclaimable page. `None` means the file has to grow. + /// Takes one reclaimable page, the lowest there is. `None` means the file has to grow. + /// + /// Lowest because [`Freelist::compact`] leaves the runs in page order, which is what lets + /// the tail empty out and the file shrink. From the **front** of the run it picks, which is + /// what keeps the entry count non-increasing and `alloc_freelist_pages` terminating. pub fn alloc(&mut self, horizon: TxnId) -> Option { let idx = self.runs.iter().position(|r| r.freed_at <= horizon)?; let run = &mut self.runs[idx]; diff --git a/crates/big-pager/tests/reclamation.rs b/crates/big-pager/tests/reclamation.rs index 5484050..8aff8be 100644 --- a/crates/big-pager/tests/reclamation.rs +++ b/crates/big-pager/tests/reclamation.rs @@ -205,6 +205,77 @@ fn truncate_tail_refuses_while_a_reader_is_alive() { assert!(store.truncate_tail().is_ok()); } +/// **The lowest reusable page is the one handed out**, which is what lets a file that has +/// churned ever get smaller. +/// +/// Ordering the freelist by generation first was correct and hopeless for the tail: writes went +/// on landing at the end of the file while holes near the front stayed holes, so nothing was +/// ever flush against EOF for `truncate_tail` to release. The reclaim rule is untouched - a run +/// newer than the horizon is still skipped - and only the choice among reusable pages changed. +/// Against the freelist itself rather than through a commit, and for the reason this file's +/// header gives about every other assertion here: a commit takes pages for its own root, +/// catalog and freelist chains before the caller's `alloc` ever sees the list, so a page number +/// observed through one says as much about the machinery as about the rule. The rule is a +/// property of this data structure, and this is where it is exact. +#[test] +fn the_lowest_reusable_page_is_allocated_first() { + let mut f = Freelist::default(); + // Freed newest-page-first, so the *oldest* generation holds the *highest* page - which is + // what the old ordering, by generation, would have handed out first. + f.push(30, 1); + f.push(20, 2); + f.push(10, 3); + f.compact(); + + assert_eq!(f.alloc(9), Some(10), "the lowest hole, not the oldest one"); + assert_eq!(f.alloc(9), Some(20)); + assert_eq!(f.alloc(9), Some(30)); + assert_eq!(f.alloc(9), None); +} + +/// And the horizon still decides *whether* a page may be reused, not just which one. A run +/// newer than the horizon is skipped however low it sits. +#[test] +fn a_page_too_new_to_reuse_is_skipped_however_low_it_is() { + let mut f = Freelist::default(); + f.push(10, 7); // low, but freed by a transaction a reader can still see + f.push(30, 1); + f.compact(); + + assert_eq!(f.alloc(3), Some(30), "the low page is not reusable yet"); + assert_eq!(f.alloc(3), None); + assert_eq!(f.alloc(7), Some(10), "and is once the horizon passes it"); +} + +/// And the point of that ordering: a file that churns comes back down instead of only ever +/// growing. Written and dropped repeatedly, then trimmed - the tail is free by then, because +/// every write went into a hole near the front rather than onto the end. +#[test] +fn a_file_that_churns_can_be_given_back_to_the_filesystem() { + let store = Store::init(MemPager::new()).unwrap(); + for i in 0..16 { + put_fragment(&store, key(i, 0)); + } + let high_water = store.metrics().page_count; + + for i in 0..16 { + drop_fragment(&store, key(i, 0)); + } + // A few more commits, which is what a live database is doing while this happens: each one + // takes its pages from the freelist, lowest first, and leaves the tail alone. + for i in 0..4 { + put_fragment(&store, key(100 + i, 0)); + } + + let released = store.truncate_tail().unwrap(); + assert!(released > 0, "the tail was free and none of it was given back"); + assert!( + store.metrics().page_count < high_water, + "the file did not shrink: {} vs {high_water}", + store.metrics().page_count + ); +} + #[test] fn readers_are_refcounted_not_flagged() { let store = Store::init(MemPager::new()).unwrap(); From 19f5f5993f0611b4558922068f5c084f0e1211fb Mon Sep 17 00:00:00 2001 From: 9bany Date: Sat, 5 Sep 2026 00:53:24 +0700 Subject: [PATCH 3/6] feat(cluster): elect the schema leader, balance, and reclaim on their own Three things a cluster could not do for itself, each behind its own flag and each off by default, plus the correctness work they turned out to need first. **The agreement had stopped failing anything over.** `promotion` guarded itself with `commit_index() != log().len() - 1`, comparing a log index with a vector position. Once anything had been compacted away the two could never agree again, so every promotion after the first compaction was refused - silently, on a cluster reporting itself healthy. Fixed with `Raft::settled()`, and the pure rule pulled out as `promotion_for` so it can be driven against a simulated agreement. Compaction also dropped the newest map and membership rather than folding them into the sentinel the follower snapshot path already builds, and the restart replay compared a vector position with the commit index; `State` now carries the seed, so a node handed a snapshot and then restarted no longer falls back to a cluster file that describes a cluster which is gone. Format `BIGRAFT3`, reading `BIGRAFT2`. **A node that had just started believed it held its leases.** `lease_at` began at zero and so does the clock it is compared against, so a fresh process read its own silence as having just been in touch - exactly a node that restarted and may already have been replaced. It is `NEVER` now. A replicated node therefore refuses its range until the first heartbeat, which is one election at boot and is what the lease always meant. **Two nodes could intern at once.** `peer_intern` and `peer_allocate` checked nothing, on the reasoning that the config decided who led - true when the leader was a name in a file, false since it became a field in the map. A coordinator one decision behind sent keys to whoever led then, and one string got two row ids, which nothing downstream can see. The request now carries who it thinks leads, the receiver checks the map, a step-down, and a lease, and the manual move stands the source down before it reads its floor rather than racing it. **Record ids are reserved before they are handed out.** The floor lived in the leader's memory, so a successor started from what was on disk and re-issued ids promised to writes that had not landed - the one failure with no second chance. The map carries a ceiling per table, raised a block of 65 536 at a time; a leader with no floor of its own starts at the ceiling. On top of that: `--balance` lets the agreement's leader take one balancing step at a time, including admitting a learner that has answered and caught up, which `add_node` had promised and nothing did. `--elect-schema-leader` lets it give the row-key namespace away after fifteen seconds of silence, with the handover rebuilt from the survivors and blocked loudly if two of them disagree about a row id. `--reclaim` hands trailing free pages back while serving. All three run on one steward thread, which is neither the agreement's - it must not block - nor a worker's. `SIGTERM` now stands the daemon down instead of killing it: `serve_while` did the right thing already and only tests called it. Known and documented rather than hidden: an automatic handover can burn row ids the dead leader assigned to writes that never landed, so a deposed node is marked behind and must be repaired before it is trusted again; and `--reclaim` releases only pages flush against the end of the file, so a database that is emptied and then left alone stays large. --- crates/big-bin/src/serve.rs | 135 +++++++- crates/big-bin/src/tty.rs | 7 +- crates/big-bin/tests/serve.rs | 112 ++++++ crates/big-cluster/src/admin.rs | 273 +++++++++++++-- crates/big-cluster/src/balance.rs | 39 ++- crates/big-cluster/src/config.rs | 19 +- crates/big-cluster/src/controller.rs | 136 +++++++- crates/big-cluster/src/counters.rs | 74 ++++ crates/big-cluster/src/error.rs | 38 ++ crates/big-cluster/src/lib.rs | 38 +- crates/big-cluster/src/ownership.rs | 81 ++++- crates/big-cluster/src/raft.rs | 199 ++++++++++- crates/big-cluster/src/wire/agreement.rs | 14 +- crates/big-cluster/src/wire/request.rs | 43 ++- crates/big-cluster/src/write.rs | 42 ++- crates/big-cluster/tests/balance.rs | 80 ++++- crates/big-cluster/tests/config.rs | 26 ++ crates/big-cluster/tests/peers.rs | 41 +++ crates/big-cluster/tests/raft.rs | 284 ++++++++++++++- crates/big-cluster/tests/rangemap.rs | 67 +++- crates/big-cluster/tests/wire.rs | 16 + crates/big-db/src/db/mod.rs | 5 + crates/big-embed/src/lib.rs | 20 ++ crates/big-http/src/lib.rs | 27 ++ crates/big-http/src/metrics.rs | 108 +++++- crates/big-http/src/routes/mod.rs | 12 +- crates/big-http/src/routes/peer.rs | 55 ++- crates/big-http/src/steward.rs | 340 ++++++++++++++++++ crates/big-http/tests/cluster.rs | 421 ++++++++++++++++++++++- crates/big-http/tests/operability.rs | 79 +++++ docs/clustering.md | 68 ++++ runbook.md | 31 ++ 32 files changed, 2816 insertions(+), 114 deletions(-) create mode 100644 crates/big-bin/tests/serve.rs create mode 100644 crates/big-http/src/steward.rs diff --git a/crates/big-bin/src/serve.rs b/crates/big-bin/src/serve.rs index 42531e0..485d0c0 100644 --- a/crates/big-bin/src/serve.rs +++ b/crates/big-bin/src/serve.rs @@ -31,6 +31,7 @@ use big_embed::Api; use big_http::{log, Auth, Server, ServerConfig}; use big_tls::TlsConfig; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; const DEFAULT_ADDR: &str = "127.0.0.1:7654"; @@ -67,6 +68,28 @@ usage: big serve [addr] [options] --queue connections allowed to wait; past this, 503 --read-timeout how long a client may take to send a request --query-timeout wall-clock budget for one query; 0 means none + --reclaim give trailing free pages back to the filesystem while serving. + Copy-on-write leaves holes and the freelist reuses them, so a + file that has churned is mostly free space it never hands back; + without this only an offline `big compact` shrinks it. Acts once + a quarter of the file is reclaimable, needs a moment with no + query in flight, and takes the write lock for one truncation. + Watch big_pages_reclaimed_total and big_reclaim_blocked_total + --elect-schema-leader let the agreement give the row-key namespace to another node + when the one holding it has been silent for fifteen seconds. + Without it, `bigctl cluster schema-leader` is the only way it + moves, and a dead leader means no *new* row key can be assigned + until it is back. With it, a failover copies every row key of + every table to the successor - and can burn row ids the dead + leader assigned to writes that never landed, so the deposed node + must be repaired before it is trusted again. See + docs/clustering.md. Needs --cluster + --balance let the cluster reshape itself: a node that is draining has its + ranges moved away, a node with nothing is given the tail, and a + node much fuller than another gives a range up. One step at a + time, decided by the agreement's leader. Off unless passed - a + cluster that reshapes itself unasked is one whose shape an + operator cannot predict. Needs --cluster to mean anything --max-row-keys refuse to invent more than n row keys; 0 means no limit every row key is resident in memory in both directions, so this is the ceiling on the one allocation that grows with @@ -157,7 +180,43 @@ pub fn main(args: &[String]) -> std::io::Result<()> { refuse_an_open_port(&server, bound, &opts); announce(&server, bound, &opts); - server.serve() + // **Stopped by a signal, not killed by one.** `SIGTERM` is what every supervisor sends + // first - `docker stop`, a pod being evicted, `systemctl stop` - and the default disposition + // ends the process mid-request: the workers die with sockets open, and the agreement learns + // this node has gone only when its heartbeats stop. Standing down instead finishes what is + // in flight, takes nothing more, and leaves the agreement before the port closes. + // + // Two things this does *not* promise, said here rather than found out. The agreement is + // stopped before the workers drain, so a request that arrives during the drain for a range + // this node was serving gets `503 not_serving` - correct, because a node standing down must + // not answer for a range it may already have lost, but not the same as "every request + // completes". And a move this node was driving as the agreement's leader is not finished + // for it: the range stays marked moving until an operator cancels or another leader + // finishes it. A signal narrows that window; it does not close it. + server.serve_while(stand_down_on_signal()) +} + +/// Installs the handlers that ask the server to stand down, and hands back the flag they clear. +/// +/// **One of the two `unsafe` blocks in this crate** - the other is `tty::Echo` - and the whole +/// of it is two `libc::signal` calls. The handler does exactly one thing: a relaxed store into a +/// static `AtomicBool`, which is on the short list of what a signal handler may do. Nothing is +/// allocated, locked, or printed from it; the accept loop notices the flag on its next poll. +#[allow(unsafe_code)] +fn stand_down_on_signal() -> &'static AtomicBool { + static RUNNING: AtomicBool = AtomicBool::new(true); + extern "C" fn stand_down(_: libc::c_int) { + RUNNING.store(false, Ordering::Relaxed); + } + // SAFETY: `stand_down` is an `extern "C"` function with the signature `signal` expects, it + // touches only a `static AtomicBool` through an atomic store (async-signal-safe), and the + // static lives for the whole program, so the handler can never run against freed memory. + // `signal` itself has no memory-safety preconditions beyond a valid handler pointer. + unsafe { + libc::signal(libc::SIGTERM, stand_down as *const () as libc::sighandler_t); + libc::signal(libc::SIGINT, stand_down as *const () as libc::sighandler_t); + } + &RUNNING } /// The certificate this listener presents, or the decision to serve in the clear. @@ -289,7 +348,20 @@ fn assemble( // two. let state = format!("{}.raft", opts.path); eprintln!("big: agreement state in {state}"); - Cluster::new(api, config, tls, Box::new(big_cluster::raft::FileStore::new(state))) + let leases = big_cluster::controller::Leases { + move_schema_after: opts + .elect_schema_leader + .then_some(big_cluster::controller::Leases::SCHEMA_FAILOVER), + ..big_cluster::controller::Leases::default() + }; + Cluster::with_timing( + api, + config, + tls, + Box::new(big_cluster::raft::FileStore::new(state)), + big_cluster::raft::Timing::default(), + leases, + ) } /// The listener's settings: the defaults, with whatever the operator overrode. @@ -303,6 +375,11 @@ fn serving(auth: Auth, tls: Option, opts: &Options) -> ServerConfig { workers: opts.workers.unwrap_or(base.workers), queue_depth: opts.queue.unwrap_or(base.queue_depth), read_timeout: opts.read_timeout.unwrap_or(base.read_timeout), + // Only the switch. The thresholds stay at what the policy ships with until somebody + // has a reason to move them; three knobs would triple what has to be tested for a + // setting nobody has yet needed to change. + balance: big_cluster::balance::Policy { enabled: opts.balance, ..base.balance }, + reclaim: opts.reclaim, ..base } } @@ -376,6 +453,39 @@ fn announce(server: &Server, bound: std::net::SocketAddr, Some(d) => eprintln!("big: POST /admin/backup writes into {d}"), None => eprintln!("big: no --backup-dir; POST /admin/backup is not configured"), } + // Printed on every start, both ways: whether a cluster was allowed to reshape itself is + // the first thing an operator asks after a range has moved, and a line that only appears + // when the flag was passed is one they have to remember the absence of. + match (opts.balance, opts.cluster.is_some()) { + (true, true) => eprintln!("big: balancer on; the agreement's leader reshapes the cluster"), + // Not refused: the flag is harmless alone, and a deployment that sets it everywhere + // and adds --cluster later should not have to change twice. But a switch that does + // nothing has to say so, or somebody will wait for it. + (true, false) => eprintln!("big: --balance without --cluster: nothing to balance"), + (false, _) => eprintln!("big: balancer off; POST /admin/cluster/rebalance is one step"), + } + // Both ways again, and this one is about the file rather than the cluster: an operator + // asking why a database is not shrinking should be able to read the answer out of a log. + if opts.reclaim { + eprintln!("big: reclaiming trailing free pages while serving, once a quarter is free"); + } else { + eprintln!("big: no --reclaim; the file gives space back only to an offline `big compact`"); + } + // Printed both ways for the reason the balancer is, and with the consequence spelled out: + // an automatic handover can burn row ids the dead leader promised to writes that never + // landed, which is not something to find out from a doc after the fact. + match (opts.elect_schema_leader, opts.cluster.is_some()) { + (true, true) => eprintln!( + "big: the agreement may move the row-key namespace after 15s of silence; a \ + deposed leader must be repaired before it is trusted again" + ), + (true, false) => { + eprintln!("big: --elect-schema-leader without --cluster: nothing to elect") + } + (false, _) => eprintln!( + "big: row-key namespace stays put; bigctl cluster schema-leader is the only mover" + ), + } if !server.config().auth.is_enabled() { eprintln!("big: no authentication; this port must not be reachable from anywhere else"); } @@ -405,6 +515,12 @@ struct Options { queue: Option, read_timeout: Option, query_timeout: Option, + /// Whether the agreement's leader may reshape the cluster on its own. + balance: bool, + /// Whether the agreement may give the row-key namespace to another node on its own. + elect_schema_leader: bool, + /// Whether this node may hand trailing free pages back to the filesystem while serving. + reclaim: bool, durability: Option, cluster: Option, node: Option, @@ -432,6 +548,9 @@ impl Default for Options { queue: None, read_timeout: None, query_timeout: None, + balance: false, + elect_schema_leader: false, + reclaim: false, durability: None, cluster: None, node: None, @@ -512,6 +631,18 @@ impl Options { out.query_timeout = (secs > 0).then(|| Duration::from_secs(secs)); i += 2; } + "--balance" => { + out.balance = true; + i += 1; + } + "--elect-schema-leader" => { + out.elect_schema_leader = true; + i += 1; + } + "--reclaim" => { + out.reclaim = true; + i += 1; + } "--durability" => { let v = value()?; out.durability = Some(big_db::Durability::parse(&v).ok_or_else(|| { diff --git a/crates/big-bin/src/tty.rs b/crates/big-bin/src/tty.rs index cb3b072..89d3308 100644 --- a/crates/big-bin/src/tty.rs +++ b/crates/big-bin/src/tty.rs @@ -86,9 +86,10 @@ struct Echo { } impl Echo { - /// **The only `unsafe` in this crate**, and the whole of it is four `libc` calls around a - /// `termios` struct. The crate is `#![deny(unsafe_code)]`, so this says so out loud rather - /// than the deny being quietly relaxed at the top of the file where nobody would see it. + /// **One of the two `unsafe` blocks in this crate** - the other installs the stop signal in + /// `serve.rs` - and the whole of it is four `libc` calls around a `termios` struct. The + /// crate is `#![deny(unsafe_code)]`, so this says so out loud rather than the deny being + /// quietly relaxed at the top of the file where nobody would see it. #[allow(unsafe_code)] #[cfg(unix)] fn off() -> io::Result { diff --git a/crates/big-bin/tests/serve.rs b/crates/big-bin/tests/serve.rs new file mode 100644 index 0000000..d998fcc --- /dev/null +++ b/crates/big-bin/tests/serve.rs @@ -0,0 +1,112 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The daemon as a process: the built `big` binary, a real port, and a real signal. +//! +//! Everything else in this crate's tests drives the server in-process, which is right for the +//! client and wrong for the two things only a process has - a command line and a signal +//! disposition. What is asserted here could not be asserted any other way: that `SIGTERM` +//! ends the daemon with an exit code rather than with a signal, and that a flag typed on the +//! command line reaches the line the daemon prints about itself. + +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +fn free_port() -> SocketAddr { + TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap() +} + +/// One request, one status line. Enough to know whether anybody is listening. +fn status_of(addr: SocketAddr, target: &str) -> Option { + let mut s = TcpStream::connect_timeout(&addr, Duration::from_millis(200)).ok()?; + s.set_read_timeout(Some(Duration::from_secs(2))).ok()?; + write!(s, "GET {target} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n").ok()?; + let mut reply = String::new(); + s.read_to_string(&mut reply).ok()?; + reply.split_whitespace().nth(1)?.parse().ok() +} + +fn waiting(what: &str, budget: Duration, mut check: impl FnMut() -> bool) { + let started = Instant::now(); + while started.elapsed() < budget { + if check() { + return; + } + std::thread::sleep(Duration::from_millis(25)); + } + panic!("gave up waiting for {what} after {:?}", started.elapsed()); +} + +/// **`SIGTERM` is a request to stand down, and it is honoured.** A supervisor sends it first - +/// `docker stop`, a pod being evicted, `systemctl stop` - and a daemon that dies of it rather +/// than answering it leaves sockets open and an agreement that only finds out from silence. +/// The tell is the exit status: a process killed by a signal has none, and one that stood down +/// has zero. +#[test] +fn sigterm_stands_the_daemon_down_rather_than_killing_it() { + let dir = tempfile::tempdir().unwrap(); + let db = dir.path().join("data.big"); + let log = std::fs::File::create(dir.path().join("stderr")).unwrap(); + let addr = free_port(); + + let mut daemon = Command::new(env!("CARGO_BIN_EXE_big")) + .arg("serve") + .arg(&db) + .arg(addr.to_string()) + // On the command line so that the line it produces can be read back below. + .arg("--balance") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::from(log)) + .spawn() + .expect("the built daemon starts"); + + waiting("the daemon to answer", Duration::from_secs(15), || { + status_of(addr, "/health") == Some(200) + }); + + let told = Command::new("kill") + .args(["-TERM", &daemon.id().to_string()]) + .status() + .expect("`kill` is on the path"); + assert!(told.success(), "the signal was sent"); + + let status = { + let started = Instant::now(); + loop { + if let Some(s) = daemon.try_wait().unwrap() { + break s; + } + assert!( + started.elapsed() < Duration::from_secs(15), + "the daemon did not stand down within fifteen seconds of SIGTERM" + ); + std::thread::sleep(Duration::from_millis(25)); + } + }; + assert_eq!( + status.code(), + Some(0), + "stood down with an exit code, not killed by the signal: {status}" + ); + assert_eq!(status_of(addr, "/health"), None, "and the port is closed"); + + // The flag reached the announcement. Alone it does nothing, and it says so rather than + // letting an operator wait for a balancer with nothing to balance. + let said = std::fs::read_to_string(dir.path().join("stderr")).unwrap(); + assert!(said.contains("big serving"), "{said}"); + assert!(said.contains("--balance without --cluster"), "{said}"); +} diff --git a/crates/big-cluster/src/admin.rs b/crates/big-cluster/src/admin.rs index e0fad56..45b214a 100644 --- a/crates/big-cluster/src/admin.rs +++ b/crates/big-cluster/src/admin.rs @@ -275,10 +275,15 @@ impl Cluster

{ fn abandon(&self, id: RangeId, why: ClusterError) -> ClusterError { match self.with_map(|m| m.cancel_move(id).map_err(|e| e.to_string())) { Ok(_) => why, - Err(e) => ClusterError::Refused(format!( - "{why} - and the move could not be called off either ({e}), so range {id} is \ - still marked as moving. `cluster cancel {id}` clears it" - )), + Err(e) => { + // Counted as well as said: an operator reads the sentence, a background loop + // drops it, and the range it names silences every balancing step from here on. + self.counters.move_cancel_failed(); + ClusterError::Refused(format!( + "{why} - and the move could not be called off either ({e}), so range {id} \ + is still marked as moving. `cluster cancel {id}` clears it" + )) + } } } @@ -345,8 +350,10 @@ impl Cluster

{ /// /// **A learner, not a voter.** A node that has just arrived holds no range and has not /// caught up on the log; counting it towards a majority would raise the bar for every - /// election while it contributed nothing to one. The balancer promotes it once it has - /// something to serve - or an operator does, with `admit`. + /// election while it contributed nothing to one. With the balancer on, the agreement's + /// leader admits it once it answers for data and holds the log to within the compaction + /// margin; otherwise an operator does, with `admit`. Either way it is given nothing to + /// serve until it counts. pub fn add_node(&self, name: &str, addr: &str) -> Result<()> { let mut next = self.members(); if let Some(existing) = next.iter_mut().find(|m| m.name == name) { @@ -467,13 +474,16 @@ impl Cluster

{ /// 1. **The keys go first**, every table's, and they are not scoped to a range - a row key /// has to mean the same number in every shard, so the successor needs the whole mapping /// or it will invent a second id for a string that already has one. - /// 2. **The floor goes with them.** A leader hands out record ids from a number held in - /// memory and not yet written anywhere; a successor that started from what is *on disk* - /// would hand out ids the old leader has already given away. This is the part that has - /// no second chance - the ids are gone by the time anybody notices. - /// 3. **Then the decision commits**, and only then does the successor answer. Until it - /// does, the old leader is still the one interning, which is what keeps the window from - /// being one in which nobody is. + /// 2. **The old leader stops.** It is told the epoch the move read and refuses to intern or + /// allocate until the map is past it. The window that opens here is one in which nobody + /// interns, and a write with a new key waits; the window it closes is one in which two + /// nodes do, which no write could ever wait out. + /// 3. **The floor goes next.** A leader hands out record ids from a number held in memory + /// and not yet written anywhere; a successor that started from what is *on disk* would + /// hand out ids the old leader has already given away. This is the part that has no + /// second chance - the ids are gone by the time anybody notices - and it is read only + /// once nothing can still be raising it. + /// 4. **Then the decision commits**, and only then does the successor answer. pub fn move_schema_leader(&self, to: &str) -> Result<()> { let target = self.node_named(to)?; let source = self.schema_leader(); @@ -489,20 +499,105 @@ impl Cluster

{ self.push_keys(target, &table.name, keys)?; } - // 2. The floor: one past the highest id the old leader has handed out, whether or not - // it has landed. Taken *after* the keys, so nothing allocated during the copy is - // missed. + // 2. The old leader stops. From here until the decision lands nobody interns, which + // is the gap this move is allowed to have; the one it is not allowed to have is two + // nodes interning at once, and a source still answering while its floor is read - or + // after, to a coordinator holding the map from before the move - is exactly that. A + // source that cannot be told stops the move: refused, not raced. + let epoch = self.map().epoch; + self.step_down(source, epoch)?; + + // 3. The floor: one past the highest id the old leader has handed out, whether or not + // it has landed. Taken *after* the keys and *after* the source stopped, so it is a + // number nothing is still adding to. let floors = self.allocation_floors(source)?; self.seed_floors(target, &floors)?; - // 3. The decision. + // 4. The decision. Ready at once: the keys and the floor went across above, by hand, + // which is the whole of what a successor has to take over. self.with_map(|m| { m.schema_leader = target; + m.schema_ready = true; Ok(()) }) .map(|_| ()) } + /// Finishes a handover the agreement decided: gives the successor every row key the + /// survivors hold, then marks it ready. + /// + /// **Run by the agreement's leader, from every node it can reach.** The old leader is not + /// among them - that is why there is a handover - so the mapping is rebuilt from the + /// copies every owner took when it was told what a key meant. A key the old leader + /// interned for a write that never landed anywhere is the one thing this cannot recover; + /// the successor will hand that row id to a different string, and the deposed node, if + /// it ever comes back, will refuse the contradiction when it is repaired. That is the + /// residual risk of an automatic failover, and it is stated in `docs/clustering.md` + /// rather than hidden. + /// + /// **A contradiction among the survivors stops it.** Two live nodes disagreeing about + /// what a row id means is a cluster that has already diverged; marking the successor + /// ready would let it pick one side and write on it. Nothing interns until somebody + /// looks, and `big_cluster_schema_handover_blocked` is what makes them. + pub fn finish_schema_handover(&self) -> Result> { + let map = self.map(); + if map.schema_ready { + return Ok(None); + } + let successor = map.schema_leader; + let this = self.config.this_index(); + let outcome = self.hand_over_to(successor, this); + self.handover_blocked.store( + matches!(&outcome, Err(e) if !e.is_unreachable()), + std::sync::atomic::Ordering::Relaxed, + ); + outcome?; + self.with_map(|m| { + if m.schema_leader != successor { + return Err("the namespace moved again while it was being handed over".into()); + } + m.schema_ready = true; + Ok(()) + })?; + Ok(Some(self.name_of_agreed(successor))) + } + + /// Every row key every reachable member holds, pushed to `successor`. + fn hand_over_to(&self, successor: usize, this: usize) -> Result<()> { + // The successor's schema has to exist first, or a key naming a field it has never + // heard of has nowhere to land. Every node carries the same schema - DDL fans out - + // so this node's is as good as any. + self.match_schema(this, successor)?; + let tables = self.pull_schema(this)?.0; + for (i, member) in self.members().iter().enumerate() { + if !member.reachable() || i == successor { + continue; + } + for table in &tables { + let keys = match self.pull_keys(i, &table.name) { + Ok(keys) => keys, + // A survivor that is not answering right now contributes nothing, and + // that is not a contradiction: the next pass will ask it again. + Err(e) if e.is_unreachable() => continue, + Err(e) => return Err(e), + }; + if !keys.is_empty() { + self.push_keys(successor, &table.name, keys)?; + } + } + } + Ok(()) + } + + /// Tells a node to stop leading the schema until the map has moved past `epoch`. + fn step_down(&self, node: usize, epoch: u64) -> Result<()> { + if node == self.config.this_index() { + self.stand_down_schema(epoch); + return Ok(()); + } + self.ask(node, path::SCHEMA_STEP_DOWN, &wire::put_u64_body(epoch), None).map(|_| ()) + } + /// One past the highest record id a node has handed out for each table, landed or not. fn allocation_floors(&self, node: usize) -> Result> { if node == self.config.this_index() { @@ -515,18 +610,74 @@ impl Cluster

{ /// This node's own floors, which only the schema leader has anything in. pub fn floors_here(&self) -> Vec<(String, RecordId)> { let allocated = self.allocated.lock().unwrap_or_else(|e| e.into_inner()); + let map = self.map(); let mut out: Vec<(String, RecordId)> = Vec::new(); for table in self.schema() { - // The greater of what is on disk and what has been promised. A leader that has - // never allocated still has a floor - it is just the data's own high-water mark. + // The greatest of what is on disk, what has been promised, and what the agreement + // has been told may be promised. A leader that has never allocated still has a + // floor - it is just the data's own high-water mark. let landed = self.api.max_record(&table.name).ok().flatten().map_or(0, |m| m.saturating_add(1)); let promised = allocated.get(&table.name).copied().unwrap_or(0); - out.push((table.name.clone(), landed.max(promised))); + let reserved = map.reserved_for(&table.name); + out.push((table.name.clone(), landed.max(promised).max(reserved))); } out } + /// Raises the agreement's ceiling on the record ids `table` may be handed, to `upto`. + /// + /// **Through the agreement's leader, whoever that is.** The schema leader and the + /// agreement's leader are two roles that usually sit on two nodes, and only the second can + /// propose - so this is one hop when they differ, amortised over a block of ids. Waits + /// until this node has applied the result: a ceiling that has been proposed and not landed + /// is one a successor could still start below. + /// + /// A cluster with no agreement has nothing to commit to and nothing to be succeeded by; + /// its floor in memory is all there is, and all there needs to be. + pub(super) fn reserve_ids(&self, table: &str, upto: RecordId) -> Result<()> { + let Some(controller) = &self.controller else { return Ok(()) }; + if controller.is_leader() { + return self.reserve_here(table, upto).map(|_| ()); + } + let Some(leader) = controller.leader() else { + return Err(ClusterError::Refused( + "no node leads the agreement right now, so no record ids can be reserved; \ + try again once an election has finished" + .to_string(), + )); + }; + let body = wire::put_floors(&[(table.to_string(), upto)]); + self.ask(leader, path::RESERVE, &body, None)?; + self.await_reserved(table, upto) + } + + /// The agreement leader's half of [`Cluster::reserve_ids`]: commits the ceiling. + /// + /// Public because the peer route calls it. Answers the epoch it landed at. + pub fn reserve_here(&self, table: &str, upto: RecordId) -> Result { + self.with_map(|m| { + m.reserve(table, upto); + Ok(()) + }) + } + + /// Waits until this node's own map carries a ceiling of at least `upto` for `table`. + fn await_reserved(&self, table: &str, upto: RecordId) -> Result<()> { + let deadline = Instant::now() + PROPOSAL_TIMEOUT; + while Instant::now() < deadline { + if self.map().reserved_for(table) >= upto { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(5)); + } + Err(ClusterError::Refused(format!( + "the agreement's leader took the reservation but it has not reached this node \ + after {}s; nothing was handed out", + PROPOSAL_TIMEOUT.as_secs() + ))) + } + /// Starts a node's floors at least as high as these. fn seed_floors(&self, node: usize, floors: &[(String, RecordId)]) -> Result<()> { if node == self.config.this_index() { @@ -567,7 +718,8 @@ impl Cluster

{ .filter_map(|t| self.api.max_record(&t.name).ok().flatten()) .max() .map_or(0, |m| m.saturating_add(1)); - crate::balance::NodeLoad { pages: Some(pages), frontier } + // This node holds its own log, whatever else is true of it. + crate::balance::NodeLoad { pages: Some(pages), frontier, caught_up: true } } /// What every node weighs, in member order. @@ -584,13 +736,21 @@ impl Cluster

{ if i == self.config.this_index() { return self.load(); } - match self.ask(i, path::LOAD, &[], None) { - Err(_) => crate::balance::NodeLoad::default(), - Ok(bytes) => match wire::get_load(&bytes) { - Err(_) => crate::balance::NodeLoad::default(), - Ok((pages, frontier)) => { - crate::balance::NodeLoad { pages: Some(pages), frontier } - } + // **A node that does not answer is `None`, not zero** - and it is counted, + // because to the balancer it is a node that can neither give nor take, and a + // transient timeout and a dead machine would otherwise be the same silence. + match self + .ask(i, path::LOAD, &[], None) + .and_then(|b| self.read(i, || wire::get_load(&b))) + { + Err(_) => { + self.counters.load_unanswered(); + crate::balance::NodeLoad::default() + } + Ok((pages, frontier)) => crate::balance::NodeLoad { + pages: Some(pages), + frontier, + caught_up: self.controller.as_ref().is_some_and(|c| c.caught_up(i)), }, } }) @@ -603,21 +763,37 @@ impl Cluster

{ /// move is a moment where a query can fail*. So this never chains two changes, and a /// cluster that needs three moves takes three calls - each one against facts gathered /// afresh, rather than against a plan made before the first move happened. - pub fn rebalance(&self, policy: &crate::balance::Policy) -> Result> { + pub fn rebalance(&self, policy: &crate::balance::Policy) -> Result> { let (map, members, loads) = (self.map(), self.members(), self.loads()); let Some(action) = crate::balance::plan(&map, &members, &loads, policy) else { return Ok(None); }; - match action { + let done = match action { + crate::balance::Action::Admit { node } => { + self.admit(&node).map(|_| Balanced::Admitted { node }) + } crate::balance::Action::SplitTail { at, to } => { - self.split_range(at, Some(&to))?; - Ok(Some(format!("split the tail at {at} and gave it to `{to}`"))) + self.split_range(at, Some(&to)).map(|_| Balanced::SplitTail { at, to }) } crate::balance::Action::Move { range, to } => { - let report = self.move_range(range, &to)?; - Ok(Some(format!("moved shards {} to `{to}`", report.shards))) + self.move_range(range, &to).map(Balanced::Moved) + } + }; + // Counted here rather than by the caller, so the operator's one step and the steward's + // many are one number - and so that a move whose source could not let go is a number + // at all, which as a field on a report handed to a loop it was not. + match &done { + Ok(Balanced::Admitted { .. }) => self.counters.balance_admit(), + Ok(Balanced::SplitTail { .. }) => self.counters.balance_split(), + Ok(Balanced::Moved(report)) => { + self.counters.balance_move(); + if !report.dropped { + self.counters.balance_drop_failed(); + } } + Err(_) => self.counters.balance_error(), } + done.map(Some) } /// What the cluster looks like right now: every range, who holds it, and what is moving. @@ -629,7 +805,7 @@ impl Cluster

{ Topology { epoch: map.epoch, leader: self.controller.as_ref().and_then(|c| c.leader()).and_then(|n| self.name_of(n)), - schema_leader: self.name_of(map.schema_leader).unwrap_or_default(), + schema_leader: self.name_of_agreed(map.schema_leader), members: self .members() .iter() @@ -680,6 +856,33 @@ pub struct MoveReport { pub outcome: String, } +/// What one balancing step did. +/// +/// A value rather than a sentence, so that the caller who wants the sentence gets the same one +/// it always got and the caller who wants to count gets something to count. The move carries +/// its whole report: `dropped` in particular is a fact a background loop must not lose. +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum Balanced { + /// A learner that had caught up was made a full member. + Admitted { node: String }, + /// The tail was cut above everything written and the empty half given to a node with + /// nothing. No bytes moved. + SplitTail { at: big_engine::ShardId, to: String }, + /// A populated range was handed to another node. + Moved(MoveReport), +} + +impl Balanced { + /// The sentence `POST /admin/cluster/rebalance` has always answered with. + pub fn describe(&self) -> String { + match self { + Self::Admitted { node } => format!("admitted `{node}` as a full member"), + Self::SplitTail { at, to } => format!("split the tail at {at} and gave it to `{to}`"), + Self::Moved(r) => format!("moved shards {} to `{}`", r.shards, r.to), + } + } +} + /// The cluster's shape, as a report. #[derive(Clone, PartialEq, Eq, Debug)] pub struct Topology { diff --git a/crates/big-cluster/src/balance.rs b/crates/big-cluster/src/balance.rs index 46818f0..fbfe9e1 100644 --- a/crates/big-cluster/src/balance.rs +++ b/crates/big-cluster/src/balance.rs @@ -43,6 +43,13 @@ pub struct NodeLoad { /// One past the highest record id the node holds, over every table. What a tail split is /// cut above. pub frontier: u64, + /// Whether the node holds the agreement's log to within a margin of its end. + /// + /// What decides whether a learner may become a member. A node still replaying what it + /// missed would raise the bar for every election while being able to help decide none - + /// the exact reason a learner is not a voter to begin with. Only the leader knows this, + /// which is fine: only the leader proposes. + pub caught_up: bool, } /// When the balancer is allowed to act, and how hard. @@ -68,6 +75,12 @@ impl Default for Policy { /// The one thing to do next. #[derive(Clone, PartialEq, Eq, Debug)] pub enum Action { + /// Make a learner that has caught up a full member, so that it votes and may hold a range. + /// + /// **A change to who the cluster is, not to who holds what** - and the only one the + /// balancer makes. It has to come first: until the node is a member, nothing below can see + /// it as a node with nothing to give it the tail. + Admit { node: String }, /// Cut the tail above everything written and give the empty half to a node with nothing. /// /// **The scale-out that moves no bytes**, and the reason it is preferred: it costs one @@ -81,10 +94,12 @@ pub enum Action { /// /// The order is the policy, and the first that applies wins: /// -/// 1. **A node that is draining** gives up a range. An operator asked for this, so it outranks +/// 1. **A learner that is answering and has caught up** is admitted. It joined to take work, +/// and nothing below can give it any until it counts. +/// 2. **A node that is draining** gives up a range. An operator asked for this, so it outranks /// anything the balancer noticed by itself. -/// 2. **A node with nothing** is given the tail, cut above everything written. No bytes move. -/// 3. **A spread that is too wide** moves one range from the fullest node to the emptiest. +/// 3. **A node with nothing** is given the tail, cut above everything written. No bytes move. +/// 4. **A spread that is too wide** moves one range from the fullest node to the emptiest. /// /// `None` whenever anything is already in flight: a second decision made about a map that has /// not settled is a decision about a state that is already gone. @@ -104,6 +119,18 @@ pub fn plan( } let name = |n: usize| members.get(n).map(|m: &Member| m.name.clone()); + + // 1. Somebody joined, and is ready to count. Answering `/internal/load` is what proves it + // is up for data and not only for the log; caught up is what proves counting it will not + // stall the next election. + for (i, m) in members.iter().enumerate() { + if !matches!(m.state, crate::raft::MemberState::Learner) { + continue; + } + if load.get(i).is_some_and(|l| l.pages.is_some() && l.caught_up) { + return Some(Action::Admit { node: name(i)? }); + } + } // A node that did not answer is not a candidate in either direction: it may be full, it may // be empty, and sending work to a machine that is not there is the worse guess. let live = |n: usize| { @@ -112,7 +139,7 @@ pub fn plan( }; let pages = |n: usize| load.get(n).and_then(|l| l.pages).unwrap_or(0); - // 1. Somebody asked for this. + // 2. Somebody asked for this. for (i, m) in members.iter().enumerate() { if !matches!(m.state, crate::raft::MemberState::Draining) { continue; @@ -124,7 +151,7 @@ pub fn plan( return Some(Action::Move { range: map.ranges[range].id, to: name(to)? }); } - // 2. A node that holds nothing. Give it the part of the space nothing has been written to + // 3. A node that holds nothing. Give it the part of the space nothing has been written to // yet, which is the only thing that can change hands without a copy. if let Some(empty) = (0..members.len()).find(|n| live(*n) && map.held_by(*n).is_empty()) { // Above every record any node holds, so the half handed over is empty by construction. @@ -138,7 +165,7 @@ pub fn plan( } } - // 3. A spread that has grown too wide. + // 4. A spread that has grown too wide. let fullest = (0..members.len()).filter(|n| live(*n)).max_by_key(|n| pages(*n))?; let emptiest = (0..members.len()).filter(|n| live(*n)).min_by_key(|n| pages(*n))?; if fullest == emptiest { diff --git a/crates/big-cluster/src/config.rs b/crates/big-cluster/src/config.rs index 5c8aec6..82468dd 100644 --- a/crates/big-cluster/src/config.rs +++ b/crates/big-cluster/src/config.rs @@ -661,7 +661,14 @@ impl ClusterConfig { text(&node.shards.to_string(), &mut h); eat(&node.replica_of.map_or(u64::MAX, |r| r as u64).to_le_bytes(), &mut h); } - text(&self.nodes[self.leader].name.clone(), &mut h); + // **The schema leader is deliberately not folded in.** It used to be, and it was right + // while the leader was a name read once from a file: two files naming different ones + // described two different clusters. It is a field in the map now, moved by an operator + // and - once a lease allows it - by the agreement itself. Two nodes whose files were + // updated at different moments after a *correct* failover would stop being able to + // talk to each other, which is a successful failover causing a partition: the worst + // failure available. The map overrides the file within a heartbeat anyway, so folding + // it in protected nothing. h } @@ -746,7 +753,15 @@ impl ClusterConfig { .collect(); // Nothing is behind before anything has happened, which is exactly what the file // asserts by naming a primary for every range. - crate::raft::RangeMap { epoch: 0, ranges, stale: Vec::new(), schema_leader: self.leader } + crate::raft::RangeMap { + epoch: 0, + ranges, + stale: Vec::new(), + schema_leader: self.leader, + reserved: Vec::new(), + // The file's leader has nothing to take over: it starts holding the namespace. + schema_ready: true, + } } /// The members the cluster starts from, in the file's order. diff --git a/crates/big-cluster/src/controller.rs b/crates/big-cluster/src/controller.rs index 8adb461..eddb6c4 100644 --- a/crates/big-cluster/src/controller.rs +++ b/crates/big-cluster/src/controller.rs @@ -56,6 +56,16 @@ pub struct Leases { /// stopped before the new one starts, and the only thing making that true is this /// inequality plus clocks that run at similar rates. pub promote_after: Duration, + /// The leader gives the row-key namespace to another node after this long without hearing + /// from the one that holds it. `None` leaves that to an operator. + /// + /// **Much longer than `promote_after`, and off by default.** Moving the namespace copies + /// every row key of every table to the successor, which is the most expensive thing the + /// agreement can decide to do - so it waits until ranges have already failed over and the + /// map has settled, and it does not happen at all unless somebody turned it on. The same + /// lease the schema leader serves under (`serve_for`) is what makes this safe: by the time + /// the agreement moves the namespace, the old holder has long since stopped acting on it. + pub move_schema_after: Option, } impl Default for Leases { @@ -63,10 +73,25 @@ impl Default for Leases { Self { serve_for: Duration::from_millis(1_500), promote_after: Duration::from_millis(4_500), + move_schema_after: None, } } } +impl Leases { + /// How long the agreement waits before moving the namespace, when it is allowed to. + /// + /// Three and a third times `promote_after`: strictly longer is the property that matters, + /// and this much longer is the margin for a range failover to have landed first. + pub const SCHEMA_FAILOVER: Duration = Duration::from_secs(15); +} + +/// The lease of a node that has not yet heard from the agreement at all. +/// +/// A sentinel rather than a zero because the clock it is compared against also starts at zero: +/// a fresh node would otherwise read its own silence as having just been in touch. +const NEVER: u64 = u64::MAX; + /// One node's half of the agreement, and the thread that drives it. pub struct Controller { raft: Mutex, @@ -95,6 +120,11 @@ pub struct Controller { roster: RwLock>, /// Milliseconds since this process started, at the last moment this node could prove it /// was still in touch with a majority. The lease, in one number. + /// + /// [`NEVER`] until it has proved that once. **Not zero**: this clock starts at zero too, so + /// zero reads as "heard from the agreement just now" on a node that has heard from nobody + /// at all - which is exactly a node that has restarted and may already have been replaced + /// while it was away. lease_at: AtomicU64, started: Instant, inbox: mpsc::Sender, @@ -211,14 +241,17 @@ impl Controller { // than taken from the file alone, or a node that restarted would come back believing // the file's map - which, once a range has moved, describes a cluster that no longer // exists. Members follow every entry because a configuration change applies when it is - // appended; ranges follow only committed ones. + // appended; ranges follow only committed ones - and "committed" is a log position, so + // the vector's position is offset by the base, or a compacted log would apply a map no + // majority had agreed to yet. let mut members = config.seed_members(); { let mut map = ranges.write().expect("no panic holds this lock"); for (i, entry) in raft.log().iter().enumerate() { + let index = raft.base() + i as u64; match &entry.decision { Decision::Members(ms) => members = ms.clone(), - Decision::Ranges(m) if i as u64 <= raft.commit_index() => *map = m.clone(), + Decision::Ranges(m) if index <= raft.commit_index() => *map = m.clone(), _ => {} } } @@ -243,7 +276,7 @@ impl Controller { this, peers: Arc::clone(&peers), roster: RwLock::new(None), - lease_at: AtomicU64::new(0), + lease_at: AtomicU64::new(NEVER), started, inbox: tx, outbox, @@ -281,8 +314,19 @@ impl Controller { /// the new member list stops waiting while the entry is still in flight, and the next /// proposal is refused as busy. Nothing is settled until the log is. pub fn settled(&self) -> bool { + self.raft.lock().expect("no panic holds this lock").settled() + } + + /// Whether `node` holds the log to within the margin a leader keeps before compacting. + /// + /// **What lets a learner become a member.** A node further behind than that would be sent + /// a snapshot rather than entries, and counting it towards a majority while it replays one + /// would raise the bar for an election it could not help decide. Only a leader tracks what + /// its followers hold, so on any other node this is `false` - which is right, because only + /// a leader proposes the change this answer gates. + pub fn caught_up(&self, node: NodeId) -> bool { let raft = self.raft.lock().expect("no panic holds this lock"); - raft.commit_index() == raft.last_index() + raft.matched(node).is_some_and(|m| m + Raft::KEEP_ENTRIES >= raft.last_index()) } /// The map, as last committed. @@ -337,8 +381,23 @@ impl Controller { map.ranges.iter().any(|r| r.group.contains(&self.this) && r.group.len() > 1) } + /// Whether this node may act as the schema leader the map names it as. + /// + /// **Always fenced.** A range with no copy is never fenced because nothing could take it; + /// the schema leader can always be replaced, so it acts only while it has heard from the + /// agreement recently enough to know it has not been. The same lease as a range, and the + /// same margin against `promote_after` is what makes replacing it safe. + pub fn may_lead_schema(&self) -> bool { + self.since_lease() < self.leases.serve_for.as_millis() as u64 + } + fn since_lease(&self) -> u64 { - self.now().saturating_sub(self.lease_at.load(Ordering::Relaxed)) + match self.lease_at.load(Ordering::Relaxed) { + // Never in touch is not "in touch a long time ago"; it is further than any lease + // reaches, which is what a node that has just started has to assume about itself. + NEVER => NEVER, + at => self.now().saturating_sub(at), + } } /// A message from a peer, on its way to the one thread that decides. @@ -475,10 +534,24 @@ impl Controller { /// already committed - a second proposal while the first is in flight would be a decision /// made about a state that has not settled. fn promotion(&self, raft: &Raft, now: u64) -> Option { - if raft.commit_index() != raft.log().len() as u64 - 1 { + Self::promotion_for(&self.leases, raft, &self.map(), now).map(Decision::Ranges) + } + + /// The rule itself, with nothing of this node's around it. + /// + /// Public so that it can be run against a simulated agreement - a `Raft` a test has driven + /// to a chosen state - without a thread, a socket or a store. The controller only decides + /// *when* to ask; what is asked is entirely here. + pub fn promotion_for( + leases: &Leases, + raft: &Raft, + current: &RangeMap, + now: u64, + ) -> Option { + if !raft.settled() { return None; } - let promote_after = self.leases.promote_after.as_millis() as u64; + let promote_after = leases.promote_after.as_millis() as u64; // A node nobody has heard from *yet* is not a node that has gone: at startup nothing // has been heard from anybody, and marking every peer behind before the first // heartbeat would mean a cluster that has to be repaired the moment it starts. @@ -487,7 +560,7 @@ impl Controller { || now.saturating_sub(raft.last_heard(node).unwrap_or(0)) < promote_after }; - let mut next = self.map(); + let mut next = current.clone(); let mut changed = false; // Marked behind, and never unmarked here: only a repair knows whether a copy has @@ -532,18 +605,57 @@ impl Controller { // is now serving a range it no longer holds. `GET /verify` is how that is found; the // mark is what makes somebody look. + // **The schema leader, last and slowest.** Silence long enough that the ranges above + // have already failed over and the map has settled: moving the namespace copies every + // row key of every table, and moving it on a blip would be the most expensive flap + // there is. Off unless a lease says how long, which is the operator's switch. + // + // The successor is chosen the way a range's is - in a fixed order, so two leaders + // elected in sequence choose the same node - from the voters that are answering and + // not marked behind. Not required to hold a range: after a rebalance a voter may hold + // none, and the namespace has nothing to do with ranges. What it is *not* given is + // the keys: this decides, and the handover that follows it - run by the agreement's + // leader from every survivor - is what makes the successor ready. Until then nobody + // interns, which is allowed; two nodes interning is not. + // + // The deposed node is marked behind whether or not it holds a copy of anything. It + // may hold row ids it interned for writes that never landed, which the survivors have + // never seen and the successor will hand out again - so it must be repaired, which + // will refuse the contradiction loudly, before it is trusted with anything. + if let Some(after) = leases.move_schema_after { + let after = after.as_millis() as u64; + let heard = |node: NodeId| { + node == raft.id() || now.saturating_sub(raft.last_heard(node).unwrap_or(0)) < after + }; + let deposed = next.schema_leader; + if !heard(deposed) { + let members = raft.membership(); + let successor = (0..members.len()) + .find(|&n| members[n].takes_ranges() && live(n) && !next.is_stale(n)); + if let Some(successor) = successor { + next.schema_leader = successor; + next.schema_ready = false; + if !next.is_stale(deposed) { + next.stale.push(deposed); + next.stale.sort_unstable(); + } + changed = true; + } + } + } + if !changed { return None; } // A promotion never reshapes the space, so this cannot fail - but the check is here // rather than assumed, because every proposal goes through it and a proposal that // skipped it would be the one that got it wrong. - if let Err(e) = next.check() { + if let Err(e) = next.check_with(raft.membership()) { debug_assert!(false, "a promotion produced an invalid map: {e}"); return None; } next.epoch += 1; - Some(Decision::Ranges(next)) + Some(next) } /// Proposes a new map, which every node will adopt once a majority has it. @@ -556,8 +668,10 @@ impl Controller { /// committed map that leaves a gap is a record id nobody answers for and there is nothing /// downstream that would notice. pub fn propose_map(&self, mut next: RangeMap) -> core::result::Result { - next.check().map_err(ProposeError::Invalid)?; let mut raft = self.raft.lock().expect("no panic holds this lock"); + // Against the membership as well as the space: a map naming a schema leader that has + // left, or one still catching up, would commit and then nobody would intern. + next.check_with(raft.membership()).map_err(ProposeError::Invalid)?; if !raft.is_leader() { return Err(ProposeError::NotLeader { leader: raft.leader() }); } diff --git a/crates/big-cluster/src/counters.rs b/crates/big-cluster/src/counters.rs index e8a4f38..ccaff82 100644 --- a/crates/big-cluster/src/counters.rs +++ b/crates/big-cluster/src/counters.rs @@ -32,6 +32,13 @@ pub struct Counters { sent: AtomicU64, unreachable: AtomicU64, refused: AtomicU64, + load_unanswered: AtomicU64, + balance_moves: AtomicU64, + balance_splits: AtomicU64, + balance_admits: AtomicU64, + balance_errors: AtomicU64, + balance_drops_failed: AtomicU64, + move_cancel_failed: AtomicU64, } impl Counters { @@ -43,6 +50,42 @@ impl Counters { self.sent.fetch_add(1, Ordering::Relaxed); } + /// A peer that did not say what it weighs. To the balancer that node is neither a source + /// nor a destination - so a count that climbs is a cluster that will not reshape, and + /// without this number a transient timeout and a dead machine look the same. + pub fn load_unanswered(&self) { + self.load_unanswered.fetch_add(1, Ordering::Relaxed); + } + + pub fn balance_move(&self) { + self.balance_moves.fetch_add(1, Ordering::Relaxed); + } + + pub fn balance_split(&self) { + self.balance_splits.fetch_add(1, Ordering::Relaxed); + } + + pub fn balance_admit(&self) { + self.balance_admits.fetch_add(1, Ordering::Relaxed); + } + + pub fn balance_error(&self) { + self.balance_errors.fetch_add(1, Ordering::Relaxed); + } + + /// A move that landed, whose source could not let go of its copy. Space nobody reclaims + /// until somebody looks; a background loop that only ever saw the move succeed would never + /// tell them to. + pub fn balance_drop_failed(&self) { + self.balance_drops_failed.fetch_add(1, Ordering::Relaxed); + } + + /// A move that failed and could not be called off either. Each one is a range still marked + /// moving, and a range marked moving silences the balancer cluster-wide. + pub fn move_cancel_failed(&self) { + self.move_cancel_failed.fetch_add(1, Ordering::Relaxed); + } + /// A peer that did not answer. Returns the error it was given, so a call site can count and /// fail in one expression rather than in two lines that can drift apart. pub fn unreachable(&self, e: E) -> E { @@ -63,6 +106,13 @@ impl Counters { sent: self.sent.load(Ordering::Relaxed), unreachable: self.unreachable.load(Ordering::Relaxed), refused: self.refused.load(Ordering::Relaxed), + load_unanswered: self.load_unanswered.load(Ordering::Relaxed), + balance_moves: self.balance_moves.load(Ordering::Relaxed), + balance_splits: self.balance_splits.load(Ordering::Relaxed), + balance_admits: self.balance_admits.load(Ordering::Relaxed), + balance_errors: self.balance_errors.load(Ordering::Relaxed), + balance_drops_failed: self.balance_drops_failed.load(Ordering::Relaxed), + move_cancel_failed: self.move_cancel_failed.load(Ordering::Relaxed), } } } @@ -75,6 +125,20 @@ pub struct Counts { pub unreachable: u64, /// How many were answered with a refusal. pub refused: u64, + /// Peers that did not say what they weigh when the balancer asked. + pub load_unanswered: u64, + /// Ranges the balancer moved. + pub balance_moves: u64, + /// Tails the balancer cut for a node with nothing. + pub balance_splits: u64, + /// Learners the balancer made full members. + pub balance_admits: u64, + /// Balancing steps that failed. + pub balance_errors: u64, + /// Moves that landed but left their source holding a copy it could not drop. + pub balance_drops_failed: u64, + /// Moves that failed and could not be called off - ranges still marked moving. + pub move_cancel_failed: u64, } /// Everything `/metrics` renders about the cluster, taken at one moment. @@ -91,5 +155,15 @@ pub struct Snapshot { pub leader: bool, /// Copies the agreement has marked behind. **Redundancy this cluster has lost.** pub behind: usize, + /// Ranges with a move in flight. **Nonzero for longer than a move takes is the balancer + /// silenced**: it plans nothing while anything is moving, and nothing clears the mark on + /// its own. + pub moving: usize, + /// Whether the schema leader has finished taking the namespace over. `false` is a window + /// in which nobody assigns row ids; one that stays `false` is a handover that is stuck. + pub schema_ready: bool, + /// Whether the last handover found two survivors disagreeing about a row id. **Somebody + /// has to look**: nothing clears this but a handover that succeeds. + pub handover_blocked: bool, pub counts: Counts, } diff --git a/crates/big-cluster/src/error.rs b/crates/big-cluster/src/error.rs index 1ff0d2c..56bb4d2 100644 --- a/crates/big-cluster/src/error.rs +++ b/crates/big-cluster/src/error.rs @@ -66,6 +66,22 @@ pub enum ClusterError { /// would say so, which is the one failure a client could never see - so a node that cannot /// prove it is still the primary stops being one. NotServing { node: String, shards: String }, + /// A request to intern or allocate reached a node that does not lead the schema. + /// + /// Either the sender holds an older map, or this node has stood down for a move that has + /// not landed yet. Retryable, against the leader the answer names: two nodes interning at + /// once hand one string two row ids, and this refusal is what keeps it to one. + NotSchemaLeader { node: String, leader: String, epoch: u64 }, + /// The map names this node as the schema leader and it has lost touch with the agreement. + /// + /// It cannot know it has not been replaced, so it stops - `NotServing`, for the one role + /// that is not a range. A write introducing a new key waits; everything else is untouched. + SchemaLeaseLost { node: String }, + /// The namespace has just moved and its new holder has not finished taking it over. + /// + /// Nobody interns in this window - the old leader has been deposed, the new one does not + /// yet hold every row key - and a write with a new key waits for it to close. Retryable. + SchemaHandover { node: String }, /// An operator asked for something the cluster will not do, and the message says why. /// /// A string rather than a variant per cause: these are read by a person at a terminal, they @@ -115,6 +131,9 @@ impl ClusterError { Self::Unreachable { .. } | Self::LeaderUnreachable { .. } | Self::NotServing { .. } + | Self::NotSchemaLeader { .. } + | Self::SchemaLeaseLost { .. } + | Self::SchemaHandover { .. } | Self::StaleRoute { .. } | Self::RangeMoving { .. } => 503, Self::Peer { status, .. } => *status, @@ -150,6 +169,7 @@ impl ClusterError { self, Self::Unreachable { .. } | Self::NotServing { .. } + | Self::SchemaLeaseLost { .. } | Self::StaleRoute { .. } | Self::RangeMoving { .. } | Self::Timeout @@ -170,6 +190,9 @@ impl ClusterError { Self::Overflow => "sum_overflow", Self::Timeout => "query_timeout", Self::NotServing { .. } => "not_serving", + Self::NotSchemaLeader { .. } => "not_schema_leader", + Self::SchemaLeaseLost { .. } => "schema_lease_lost", + Self::SchemaHandover { .. } => "schema_handover", Self::Refused(_) => "refused", Self::StaleRoute { .. } => "stale_route", Self::RangeMoving { .. } => "range_moving", @@ -204,6 +227,21 @@ impl core::fmt::Display for ClusterError { } Self::Overflow => write!(f, "the totals from two nodes do not fit in one number"), Self::Timeout => write!(f, "the request ran out of time"), + Self::NotSchemaLeader { node, leader, epoch } => write!( + f, + "`{node}` does not lead the schema; as of epoch {epoch} that is {leader}. Ask \ + there instead" + ), + Self::SchemaLeaseLost { node } => write!( + f, + "`{node}` leads the schema and has lost touch with the agreement, so it cannot \ + know it still does; a key it has never assigned a row id waits until it can" + ), + Self::SchemaHandover { node } => write!( + f, + "`{node}` was just given the schema and has not finished taking it over; a key \ + nobody has assigned a row id waits until it has" + ), Self::NotServing { node, shards } => write!( f, "`{node}` holds shards {shards} and has lost touch with the agreement, so it \ diff --git a/crates/big-cluster/src/lib.rs b/crates/big-cluster/src/lib.rs index f717f1b..6f12275 100644 --- a/crates/big-cluster/src/lib.rs +++ b/crates/big-cluster/src/lib.rs @@ -71,7 +71,7 @@ mod admin; pub mod balance; -pub use admin::{MemberReport, MoveReport, RangeReport, Topology}; +pub use admin::{Balanced, MemberReport, MoveReport, RangeReport, Topology}; mod ddl; // The one item a sibling borrows across the split: `repair` recreates a field exactly as // another node has it, and that is a `Ddl` rather than a repair concern. @@ -148,9 +148,15 @@ use std::time::{Duration, Instant}; /// its data twice, so `Count` would silently double. That is precisely the quiet mistake a /// version exists to turn into a refusal. /// +/// `7` since a request to the schema leader says **who it thinks leads**. `InternRequest` and +/// `AllocateRequest` each gained an assumption at the end, and `/internal/schema/step-down` +/// was added. A `6` leader asked by a `7` coordinator would refuse the trailing bytes as +/// unreadable rather than intern for a coordinator it may no longer lead - the right failure, +/// and the version makes it a handshake failure instead of a per-request one. +/// /// The retired numbers are **not** reused. A stale peer that somehow got past the handshake /// would then misparse rather than fail, which is what `finished` exists to prevent. -pub const WIRE_VERSION: u32 = 6; +pub const WIRE_VERSION: u32 = 7; /// The header carrying [`WIRE_VERSION`]. pub const WIRE_HEADER: &str = "x-big-wire"; @@ -188,6 +194,12 @@ pub mod path { /// the row-key namespace changes hands. pub const FLOORS: &str = "/internal/floors"; pub const FLOORS_PUT: &str = "/internal/floors/put"; + /// Stop interning: the namespace is being handed over. Asked of the source before its + /// floor is read, so that the floor is final rather than racing. + pub const SCHEMA_STEP_DOWN: &str = "/internal/schema/step-down"; + /// Raise the agreement's ceiling on record ids for a table. Asked of the agreement's + /// leader by a schema leader that is not it. + pub const RESERVE: &str = "/internal/reserve"; } /// One node, playing whichever of the three roles a given request needs. @@ -219,6 +231,19 @@ pub struct Cluster { /// Not persisted, and does not need to be. A leader that restarts re-derives the first term /// from the data itself, and the floor only ever has to outlive the writes it is ahead of. allocated: std::sync::Mutex>, + /// One past the map epoch at which this node was last told to stop leading the schema. + /// Zero for never. + /// + /// A move of the namespace happens in steps, and between "stop" and the decision landing + /// this node still leads by the map and must not act like it - a source that went on + /// interning while its floor was being read would be the second interner the whole move + /// exists to prevent. Compared against the map's epoch, so leadership that comes back at + /// a later epoch is leadership again. + stood_down_at: std::sync::atomic::AtomicU64, + /// Whether the last attempt to hand the namespace to an elected successor found two + /// survivors disagreeing about what a row id means. Nobody interns while this is set; + /// `/metrics` is how somebody finds out. + handover_blocked: std::sync::atomic::AtomicBool, /// The agreement, when there is anything to agree about. /// /// `None` when no range has a copy: a range of one cannot fail over to anything, so @@ -324,6 +349,8 @@ impl Cluster

{ ranges, counters: counters::Counters::new(), allocated: Default::default(), + stood_down_at: Default::default(), + handover_blocked: Default::default(), }) } @@ -335,7 +362,9 @@ impl Cluster

{ None => (0, false), Some(c) => (c.term(), c.is_leader()), }; - let behind = self.map().stale.len(); + let map = self.map(); + let behind = map.stale.len(); + let moving = map.ranges.iter().filter(|r| r.moving.is_some()).count(); counters::Snapshot { nodes: self.config.nodes().len(), peers: self.config.nodes().len() - 1, @@ -344,6 +373,9 @@ impl Cluster

{ term, leader, behind, + moving, + schema_ready: map.schema_ready, + handover_blocked: self.handover_blocked.load(std::sync::atomic::Ordering::Relaxed), counts: self.counters.read(), } } diff --git a/crates/big-cluster/src/ownership.rs b/crates/big-cluster/src/ownership.rs index 849e1ec..d72b9db 100644 --- a/crates/big-cluster/src/ownership.rs +++ b/crates/big-cluster/src/ownership.rs @@ -75,11 +75,90 @@ impl Cluster

{ self.ranges.read().expect("no panic holds this lock").schema_leader } - /// Whether this node is the one that assigns row ids. + /// Whether the map names this node as the one that assigns row ids. + /// + /// The map's answer and nothing more. Whether this node may *act* on it is + /// [`Cluster::guard_schema`], which also asks whether it has stood down and whether it has + /// heard from the agreement recently enough to know it still leads. pub(super) fn leads_schema(&self) -> bool { self.schema_leader() == self.config.this_index() } + /// Who leads the schema and at which epoch the map said so, read together so that a + /// request built from them carries one consistent assumption. + pub(super) fn schema_lead(&self) -> (usize, u64) { + let map = self.ranges.read().expect("no panic holds this lock"); + (map.schema_leader, map.epoch) + } + + /// The refusal this node owes a request to intern or allocate when it may not. + /// + /// Three ways to owe one. **The map names somebody else**: a coordinator holding an older + /// map sent this here, and the answer says where to send it instead. **This node was told + /// to stand down** by a move that has not committed yet, so by the map it still leads and + /// by the move it no longer does. **This node has lost touch with the agreement**, so it + /// cannot know that it has not been replaced - the same reason a range stops being served, + /// and here there is no unfenced case: the schema leader can always be replaced. + /// + /// Two nodes interning at once hand one string two row ids, which nothing downstream can + /// see. Every refusal here is worth that. + pub fn guard_schema(&self) -> Result<()> { + let (leader, epoch) = self.schema_lead(); + let this = self.config.this_index(); + let node = self.config.this().name.clone(); + if leader != this { + return Err(ClusterError::NotSchemaLeader { + node, + leader: self.describe(leader), + epoch, + }); + } + if self.stood_down_at.load(std::sync::atomic::Ordering::Relaxed) > epoch { + return Err(ClusterError::NotSchemaLeader { + node, + leader: "a node this one is handing the namespace to".to_string(), + epoch, + }); + } + if !self.ranges.read().expect("no panic holds this lock").schema_ready { + return Err(ClusterError::SchemaHandover { node }); + } + if !self.controller.as_ref().is_none_or(|c| c.may_lead_schema()) { + return Err(ClusterError::SchemaLeaseLost { node }); + } + Ok(()) + } + + /// Refuses a request to the schema leader whose assumptions this node does not share. + /// + /// The one assumption a sender can state is who it thinks leads. A sender that names + /// another node reached this one by a map it has not yet applied - or by one this node has + /// not - and is told who leads *here*, so that it can ask the right node rather than have + /// this one answer a question that is no longer its to answer. + pub fn check_schema_lead(&self, led: Option<&wire::Led>) -> Result<()> { + if let Some(led) = led { + if led.leader != self.config.this_index() { + let (leader, epoch) = self.schema_lead(); + return Err(ClusterError::NotSchemaLeader { + node: self.config.this().name.clone(), + leader: self.describe(leader), + epoch, + }); + } + } + self.guard_schema() + } + + /// Stops this node interning until the map has moved past `epoch`. + /// + /// What a move of the namespace asks of its source before it reads the floor: after this + /// the floor is final rather than racing, and the window between here and the decision + /// landing is one in which nobody interns - which is allowed - rather than one in which + /// two nodes do. + pub fn stand_down_schema(&self, epoch: u64) { + self.stood_down_at.fetch_max(epoch.saturating_add(1), std::sync::atomic::Ordering::Relaxed); + } + /// A node's name, or `None` for an index the cluster file never had. /// /// An option rather than an index, because a node can now join at runtime: a `NodeId` from diff --git a/crates/big-cluster/src/raft.rs b/crates/big-cluster/src/raft.rs index 7833703..93d122a 100644 --- a/crates/big-cluster/src/raft.rs +++ b/crates/big-cluster/src/raft.rs @@ -124,6 +124,23 @@ pub struct RangeMap { /// The node that owns the row-key namespace. In the map rather than the file because /// draining the node holding it has to be able to move it. pub schema_leader: NodeId, + /// One past the highest record id the schema leader has been given leave to hand out, per + /// table. Sorted by table, so that two leaders building the same map build the same bytes. + /// + /// **The durable half of the allocation floor.** A leader hands out ids from a number in + /// its memory; a successor that started from what is on disk would hand out ids the old + /// leader had already promised to a write that had not landed - the one failure with no + /// second chance. So a leader commits a ceiling here before it hands out anything under + /// it, in blocks, and a successor starts at the ceiling. What a dead leader promised is + /// below it by construction. + pub reserved: Vec<(String, u64)>, + /// Whether the schema leader has finished taking the namespace over. + /// + /// `false` from the moment leadership moves until the successor has reconciled every row + /// key it could reach. While it is false nobody interns - not the old leader, which has + /// been deposed, and not the new one, which does not yet hold the mapping. A write with a + /// new key waits, which is allowed; two nodes interning at once is not. + pub schema_ready: bool, } /// Why a proposed map was refused. @@ -134,15 +151,44 @@ pub struct RangeMap { #[derive(Clone, PartialEq, Eq, Debug)] pub enum MapError { Empty, - NotStartingAtZero { start: ShardId }, - Unordered { at: usize }, - Overlap { from: ShardId, to: ShardId }, - Gap { from: ShardId, to: ShardId }, - NotTotal { from: ShardId }, - EmptyGroup { id: RangeId }, - PrimaryNotInGroup { id: RangeId, primary: NodeId }, - DuplicateId { id: RangeId }, - NoSuchRange { id: RangeId }, + NotStartingAtZero { + start: ShardId, + }, + Unordered { + at: usize, + }, + Overlap { + from: ShardId, + to: ShardId, + }, + Gap { + from: ShardId, + to: ShardId, + }, + NotTotal { + from: ShardId, + }, + EmptyGroup { + id: RangeId, + }, + PrimaryNotInGroup { + id: RangeId, + primary: NodeId, + }, + DuplicateId { + id: RangeId, + }, + NoSuchRange { + id: RangeId, + }, + /// The schema leader names a node the cluster does not have, or one that has left. + SchemaLeaderNotAMember { + node: NodeId, + }, + /// The schema leader names a node still catching up on the log. + SchemaLeaderIsALearner { + node: NodeId, + }, } impl core::fmt::Display for MapError { @@ -169,11 +215,58 @@ impl core::fmt::Display for MapError { } Self::DuplicateId { id } => write!(f, "two ranges are both called {id}"), Self::NoSuchRange { id } => write!(f, "there is no range {id}"), + Self::SchemaLeaderNotAMember { node } => write!( + f, + "the schema leader is node {node}, which is not in the cluster; nobody would \ + assign row ids" + ), + Self::SchemaLeaderIsALearner { node } => write!( + f, + "the schema leader is node {node}, which is still catching up on the log and \ + cannot be trusted with the row-key namespace yet" + ), } } } impl RangeMap { + /// One past the highest record id the schema leader may hand out for `table` without + /// asking the agreement first. Zero for a table nothing has been reserved for. + pub fn reserved_for(&self, table: &str) -> u64 { + self.reserved.iter().find(|(t, _)| t == table).map_or(0, |(_, upto)| *upto) + } + + /// Raises the reservation for `table` to `upto`, never lowering it. + /// + /// Never lowering is what makes it safe to apply twice, and what makes a block that was + /// reserved and never used harmless: a reservation is a promise about ids that *may* have + /// been handed out, and the higher of two promises keeps both. + pub fn reserve(&mut self, table: &str, upto: u64) { + match self.reserved.binary_search_by(|(t, _)| t.as_str().cmp(table)) { + Ok(i) => self.reserved[i].1 = self.reserved[i].1.max(upto), + Err(i) => self.reserved.insert(i, (table.to_string(), upto)), + } + } + + /// [`RangeMap::check`], and then whether the schema leader is a node that can hold the + /// namespace: a member that has not left, and not a learner. + /// + /// A second entry point because the map alone cannot tell: `schema_leader` is an index + /// into a membership the map does not carry. Every proposal goes through this one; + /// `check` is what a map can say about itself. + pub fn check_with(&self, members: &[Member]) -> core::result::Result<(), MapError> { + self.check()?; + let node = self.schema_leader; + match members.get(node) { + Some(m) if !m.reachable() => Err(MapError::SchemaLeaderNotAMember { node }), + Some(m) if matches!(m.state, MemberState::Learner) => { + Err(MapError::SchemaLeaderIsALearner { node }) + } + Some(_) => Ok(()), + None => Err(MapError::SchemaLeaderNotAMember { node }), + } + } + /// Whether this map covers the shard space exactly once, and every range is held. /// /// **Checked on every proposal rather than only at startup.** A file was read once and @@ -821,6 +914,11 @@ impl Raft { if !state.log.is_empty() { self.log = state.log; } + // The seed a snapshot handed over outranks the file's: empty is a file from before it + // was written down, and those nodes fell back to the file, which this still does. + if !state.seed.is_empty() { + self.seed = state.seed; + } // **Before the commit index is used**, because who is in the cluster is a property of // the log this node came back holding - not of the file it was first started with, // which may describe a cluster that no longer exists. @@ -842,6 +940,7 @@ impl Raft { commit: self.commit, base: self.base, log: self.log.clone(), + seed: self.seed.clone(), } } @@ -907,6 +1006,23 @@ impl Raft { .unwrap_or(0) .min(self.applied); let target = safe.saturating_sub(keep); + // **Never past the newest map or the newest membership.** The state machine is rebuilt + // by replaying the log, so the entry that last decided each has to stay in it. A + // follower handed a snapshot folds the state into `log[0]`; this is the leader's + // equivalent, and it costs at most two entries, for ever. Without it a leader that had + // compacted past its last decision came back from a restart with the file's map and + // the file's members - a cluster that, once anything had moved or joined, was gone. + let anchor = |want: fn(&Decision) -> bool| { + self.log.iter().rposition(|e| want(&e.decision)).map(|at| self.base + at as Index) + }; + let keep_from = [ + anchor(|d| matches!(d, Decision::Ranges(_))), + anchor(|d| matches!(d, Decision::Members(_))), + ] + .into_iter() + .flatten() + .min(); + let target = keep_from.map_or(target, |k| target.min(k)); if target <= self.base { return; } @@ -989,10 +1105,26 @@ impl Raft { self.heard.get(&node).copied() } + /// The highest index this node knows `node` has stored. **Leader's knowledge**: a follower + /// tracks nobody, so this is `None` everywhere but on the node that sends the appends. + pub fn matched(&self, node: NodeId) -> Option { + self.matched.get(&node).copied() + } + pub fn last_index(&self) -> Index { self.base + self.log.len() as Index - 1 } + /// Whether everything this node holds has been committed - the log has settled. + /// + /// Both sides are log positions. `commit` is one already; the last index is + /// `base + len - 1`, and the base is the part that was forgotten once, in a guard that + /// compared `commit` with the vector's length and refused every promotion after the first + /// compaction. + pub fn settled(&self) -> bool { + self.commit == self.last_index() + } + fn last_term(&self) -> Term { self.log.last().map_or(0, |e| e.term) } @@ -1484,6 +1616,14 @@ pub struct State { /// The index of `log[0]`. Zero until something has been compacted away. pub base: Index, pub log: Vec, + /// The cluster this node was started with, before the log said otherwise. + /// + /// Written down because a snapshot replaces it: a follower handed the state by its leader + /// takes the members that came with it as its seed, and a restart that forgot them would + /// fall back to the file - which, once anybody has joined or left, describes a cluster that + /// no longer exists. Empty in a file written before this was recorded, and empty means "the + /// file's", which is exactly what those nodes did. + pub seed: Vec, } pub trait Store: Send + Sync { @@ -1525,7 +1665,12 @@ impl FileStore { /// Bumped with the shape of the file. A `BIGRAFT1` file has a different header and different /// decision tags, and reading one as this format would take a count for a term. -const MAGIC: &[u8; 8] = b"BIGRAFT2"; +/// +/// `BIGRAFT3` appends the seed membership after the log. A `BIGRAFT2` file is the same bytes +/// without it, and is still read: a node upgraded in place has a file from before the seed was +/// recorded, and refusing it would be refusing the vote it already cast. +const MAGIC: &[u8; 8] = b"BIGRAFT3"; +const PREVIOUS: &[u8; 8] = b"BIGRAFT2"; impl Store for FileStore { fn load(&self) -> std::io::Result> { @@ -1662,9 +1807,20 @@ fn put_range_map(out: &mut Vec, m: &RangeMap) { for n in &m.stale { put_u64(out, *n as u64); } + put_u64(out, m.reserved.len() as u64); + for (table, upto) in &m.reserved { + put_str(out, table); + put_u64(out, *upto); + } + out.push(m.schema_ready as u8); } -fn get_range_map(b: &mut Bytes<'_>) -> core::result::Result { +/// `reservations` is whether the bytes carry the two fields `BIGRAFT3` added after `stale`. A +/// `BIGRAFT2` map has neither: nothing was reserved, and its leader was ready. +fn get_range_map( + b: &mut Bytes<'_>, + reservations: bool, +) -> core::result::Result { let epoch = b.u64()?; let schema_leader = b.u64()? as NodeId; let ranges = b.list(|b| { @@ -1682,7 +1838,12 @@ fn get_range_map(b: &mut Bytes<'_>) -> core::result::Result, members: &[Member]) { @@ -1739,13 +1900,16 @@ fn encode_state(state: &State) -> Vec { } } } + put_members(&mut out, &state.seed); out } fn decode_state(bytes: &[u8]) -> core::result::Result { - if bytes.get(..8) != Some(MAGIC) { - return Err("not a raft state file"); - } + let carries_seed = match bytes.get(..8) { + Some(m) if m == MAGIC => true, + Some(m) if m == PREVIOUS => false, + _ => return Err("not a raft state file"), + }; let mut b = Bytes { b: bytes, at: 8 }; let term = b.u64()?; let voted = b.u64()?; @@ -1756,16 +1920,17 @@ fn decode_state(bytes: &[u8]) -> core::result::Result { let term = b.u64()?; let decision = match b.byte()? { 0 => Decision::Noop, - 1 => Decision::Ranges(get_range_map(b)?), + 1 => Decision::Ranges(get_range_map(b, carries_seed)?), 2 => Decision::Members(get_members(b)?), _ => return Err("unknown decision"), }; Ok(Entry { term, decision }) })?; + let seed = if carries_seed { get_members(&mut b)? } else { Vec::new() }; if b.at != bytes.len() { return Err("bytes after the end"); } - Ok(State { term, voted_for, commit, base, log }) + Ok(State { term, voted_for, commit, base, log, seed }) } /// The members whose agreement counts, by index. diff --git a/crates/big-cluster/src/wire/agreement.rs b/crates/big-cluster/src/wire/agreement.rs index 8f1ca62..acd2d91 100644 --- a/crates/big-cluster/src/wire/agreement.rs +++ b/crates/big-cluster/src/wire/agreement.rs @@ -179,6 +179,12 @@ fn put_range_map(out: &mut Vec, m: &raft::RangeMap) { for n in &m.stale { put_u64(out, *n as u64); } + put_count(out, m.reserved.len()); + for (table, upto) in &m.reserved { + put_str(out, table); + put_u64(out, *upto); + } + put_bool(out, m.schema_ready); } fn get_range_map(r: &mut Reader<'_>) -> Result { @@ -221,7 +227,13 @@ fn get_range_map(r: &mut Reader<'_>) -> Result { for _ in 0..count { stale.push(r.u64()? as raft::NodeId); } - Ok(raft::RangeMap { epoch, ranges, stale, schema_leader }) + let count = r.count()?; + let mut reserved = Vec::with_capacity(count); + for _ in 0..count { + reserved.push((r.str()?, r.u64()?)); + } + let schema_ready = r.bool()?; + Ok(raft::RangeMap { epoch, ranges, stale, schema_leader, reserved, schema_ready }) } fn put_members(out: &mut Vec, members: &[raft::Member]) { diff --git a/crates/big-cluster/src/wire/request.rs b/crates/big-cluster/src/wire/request.rs index 15589f1..980c298 100644 --- a/crates/big-cluster/src/wire/request.rs +++ b/crates/big-cluster/src/wire/request.rs @@ -183,12 +183,45 @@ impl RecordsRequest { } } +/// What a request to the schema leader assumed about who leads, so that the receiver can +/// disagree with it. +/// +/// The routed envelope's shape, for the one role that is not a range. A coordinator holding a +/// map one decision old sends its keys to whoever led *then*; a receiver that no longer leads +/// answers with who does rather than interning anyway, which is the two-interners failure that +/// hands one string two row ids and nothing downstream can see. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct Led { + /// The map epoch the sender read the leader from. + pub epoch: u64, + /// Who the sender believes leads the schema. + pub leader: usize, +} + +pub fn put_led(out: &mut Vec, led: Option<&Led>) { + put_bool(out, led.is_some()); + let Some(led) = led else { return }; + put_u64(out, led.epoch); + put_u64(out, led.leader as u64); +} + +pub fn get_led(r: &mut Reader<'_>) -> Result> { + if !r.bool()? { + return Ok(None); + } + let epoch = r.u64()?; + let leader = r.u64()? as usize; + Ok(Some(Led { epoch, leader })) +} + /// `POST /internal/intern`: what do these keys mean. Only the schema leader answers. #[derive(Clone, PartialEq, Eq, Debug)] pub struct InternRequest { pub table: String, pub field: String, pub keys: Vec, + /// Who the sender thinks leads. `None` from a caller that made no assumption. + pub led: Option, } impl InternRequest { @@ -200,6 +233,7 @@ impl InternRequest { for k in &self.keys { put_str(&mut out, k); } + put_led(&mut out, self.led.as_ref()); out } @@ -212,8 +246,9 @@ impl InternRequest { for _ in 0..n { keys.push(r.str()?); } + let led = get_led(&mut r)?; finished(&r)?; - Ok(Self { table, field, keys }) + Ok(Self { table, field, keys, led }) } } @@ -227,6 +262,8 @@ pub struct AllocateRequest { pub table: String, /// How many consecutive ids the caller needs. A whole statement asks once. pub count: u64, + /// Who the sender thinks leads. `None` from a caller that made no assumption. + pub led: Option, } impl AllocateRequest { @@ -234,6 +271,7 @@ impl AllocateRequest { let mut out = Vec::new(); put_str(&mut out, &self.table); put_u64(&mut out, self.count); + put_led(&mut out, self.led.as_ref()); out } @@ -241,8 +279,9 @@ impl AllocateRequest { let mut r = Reader::new(bytes); let table = r.str()?; let count = r.u64()?; + let led = get_led(&mut r)?; finished(&r)?; - Ok(Self { table, count }) + Ok(Self { table, count, led }) } } diff --git a/crates/big-cluster/src/write.rs b/crates/big-cluster/src/write.rs index a8de765..4f4bd54 100644 --- a/crates/big-cluster/src/write.rs +++ b/crates/big-cluster/src/write.rs @@ -28,6 +28,14 @@ use super::*; /// node that did not propose it. const STALE_ROUTE_PAUSE_MS: u64 = 500; +/// How many record ids the schema leader reserves with the agreement at a time. +/// +/// A ceiling committed per statement would put an agreement round trip in front of every +/// `INSERT`; a block this size puts one in front of every sixty-five thousand. What a block +/// costs is that a leader replaced mid-block leaves a gap of unused ids behind it - and record +/// ids are sixty-four bits wide and shard-mapped, so a gap costs nothing at all. +const RESERVE_BLOCK: u64 = 65_536; + impl Cluster

{ /// Writes a batch, splitting it by the owner of each record. /// @@ -297,11 +305,13 @@ impl Cluster

{ /// /// Answers the first id of the run; the caller takes `count` consecutive ids from it. pub(super) fn allocate(&self, table: &str, count: u64) -> Result { - if self.leads_schema() { + let (leader, epoch) = self.schema_lead(); + if leader == self.config.this_index() { + self.guard_schema()?; return self.allocate_here(table, count); } - let leader = self.schema_leader(); - let body = wire::AllocateRequest { table: table.to_string(), count }.encode(); + let led = Some(wire::Led { epoch, leader }); + let body = wire::AllocateRequest { table: table.to_string(), count, led }.encode(); let bytes = self.ask(leader, path::ALLOCATE, &body, None).map_err(|e| match e { ClusterError::Unreachable { node, why, .. } => { ClusterError::LeaderUnreachable { node, why } @@ -326,8 +336,21 @@ impl Cluster

{ // released before the floor was raised would be a lock that decided nothing. let mut floor = self.allocated.lock().unwrap_or_else(|e| e.into_inner()); let anywhere = self.next_record(table)?; - let from = anywhere.max(floor.get(table).copied().unwrap_or(0)); - floor.insert(table.to_string(), from.saturating_add(count)); + let ceiling = self.map().reserved_for(table); + // A leader with no floor of its own for this table is a fresh one - just elected, or + // just restarted - and starts at the ceiling: everything below it may have been + // promised by whoever led before, to a write that has not landed, and the number that + // would have said so died with them. + let promised = floor.get(table).copied().unwrap_or(ceiling); + let from = anywhere.max(promised); + let end = from.saturating_add(count); + if end > ceiling { + // **Committed before it is handed out.** A block, not the run: one entry in the + // agreement per `RESERVE_BLOCK` ids rather than per statement, and a block a dead + // leader never finished is a gap in the ids, which cost nothing. + self.reserve_ids(table, from.saturating_add(count.max(RESERVE_BLOCK)))?; + } + floor.insert(table.to_string(), end); Ok(from) } @@ -375,14 +398,19 @@ impl Cluster

{ /// Not queued, not assigned locally and reconciled later: two row ids for one string is a /// silently wrong answer, and a refusal is not. pub(super) fn intern(&self, table: &str, field: &str, keys: &[&str]) -> Result> { - if self.leads_schema() { + let (leader, epoch) = self.schema_lead(); + if leader == self.config.this_index() { + // Named by the map is not the same as allowed to act: a node that has stood down + // for a move, or has lost touch with the agreement, refuses here exactly as it + // would refuse a peer. + self.guard_schema()?; return Ok(self.api.intern_keys(table, field, keys)?); } - let leader = self.schema_leader(); let body = wire::InternRequest { table: table.to_string(), field: field.to_string(), keys: keys.iter().map(|k| (*k).to_string()).collect(), + led: Some(wire::Led { epoch, leader }), } .encode(); let bytes = self.ask(leader, path::INTERN, &body, None).map_err(|e| match e { diff --git a/crates/big-cluster/tests/balance.rs b/crates/big-cluster/tests/balance.rs index 939803a..5729f6b 100644 --- a/crates/big-cluster/tests/balance.rs +++ b/crates/big-cluster/tests/balance.rs @@ -35,17 +35,89 @@ fn range(id: u64, start: u64, end: Option, primary: usize) -> Range { } fn map(ranges: Vec) -> RangeMap { - RangeMap { epoch: 1, ranges, stale: Vec::new(), schema_leader: 0 } + RangeMap { + epoch: 1, + ranges, + stale: Vec::new(), + schema_leader: 0, + reserved: Vec::new(), + schema_ready: true, + } } +/// Every node that answered has also caught up on the log, which is the common case. fn load(pages: &[Option], frontier: u64) -> Vec { - pages.iter().map(|p| NodeLoad { pages: *p, frontier }).collect() + pages.iter().map(|p| NodeLoad { pages: *p, frontier, caught_up: p.is_some() }).collect() +} + +/// The same, with node `n` still replaying what it missed. +fn behind(mut loads: Vec, n: usize) -> Vec { + loads[n].caught_up = false; + loads } fn on() -> Policy { Policy { enabled: true, ..Policy::default() } } +// ------------------------------------------------------------------------------------------- +// Admitting a node that joined +// ------------------------------------------------------------------------------------------- + +/// **A learner that is answering and has caught up is made a member, before anything else.** +/// Until it is a voter the balancer cannot see it as a node with nothing, so admitting it has +/// to come first - even ahead of a drain an operator asked for, which only needs somewhere to +/// send the range and will have one more place the moment this lands. +#[test] +fn a_learner_that_answers_and_has_caught_up_is_admitted_before_anything_else() { + let m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 1)]); + let mut members = voters(&["a", "b"]); + members.push(member("d", MemberState::Learner)); + let l = load(&[Some(100_000), Some(100_000), Some(0)], 0); + assert_eq!(plan(&m, &members, &l, &on()), Some(Action::Admit { node: "d".to_string() })); + + // Ahead of the drain: `b` wants to go, and it will have `d` to go to once `d` counts. + members[1].state = MemberState::Draining; + assert_eq!( + plan(&m, &members, &l, &on()), + Some(Action::Admit { node: "d".to_string() }), + "a member is worth more than a move, and the move comes right after" + ); +} + +/// A learner that did not say what it weighs is a learner that may not be up for data at all. +/// Replicating the log to it proves nothing about whether it can serve a range. +#[test] +fn a_learner_that_did_not_answer_is_not_admitted() { + let m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 1)]); + let mut members = voters(&["a", "b"]); + members.push(member("d", MemberState::Learner)); + let l = load(&[Some(100_000), Some(100_000), None], 0); + assert_eq!(plan(&m, &members, &l, &on()), None); +} + +/// A learner still replaying the log would raise the bar for every election while being able +/// to help decide none - the exact reason a learner is not a voter in the first place. +#[test] +fn a_learner_still_catching_up_is_not_admitted() { + let m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 1)]); + let mut members = voters(&["a", "b"]); + members.push(member("d", MemberState::Learner)); + let l = behind(load(&[Some(100_000), Some(100_000), Some(0)], 0), 2); + assert_eq!(plan(&m, &members, &l, &on()), None); +} + +/// Admitting is a change to the map's membership, and the one-at-a-time rule covers it too. +#[test] +fn nothing_is_admitted_while_something_is_moving() { + let mut m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 1)]); + m.ranges[0].moving = Some(Move { target: 1, state: MoveState::Seeding }); + let mut members = voters(&["a", "b"]); + members.push(member("d", MemberState::Learner)); + let l = load(&[Some(100_000), Some(100_000), Some(0)], 0); + assert_eq!(plan(&m, &members, &l, &on()), None); +} + // ------------------------------------------------------------------------------------------- // When it does nothing // ------------------------------------------------------------------------------------------- @@ -137,13 +209,13 @@ fn a_draining_node_is_not_given_anything() { assert_eq!(plan(&m, &members, &l, &on()), None, "`b` holds nothing and still gets nothing"); } -/// A learner is still catching up: it replicates and does not yet take ranges. +/// A learner is still catching up: it replicates, is not yet admitted, and takes no range. #[test] fn a_learner_is_not_given_a_range_until_it_is_admitted() { let m = map(vec![range(0, 0, Some(64), 0), range(1, 64, None, 0)]); let mut members = voters(&["a", "b"]); members[1].state = MemberState::Learner; - let l = load(&[Some(100_000), Some(0)], 70 * (1 << 20)); + let l = behind(load(&[Some(100_000), Some(0)], 70 * (1 << 20)), 1); assert_eq!(plan(&m, &members, &l, &on()), None); // Admitted, and now it is the obvious place for the tail. diff --git a/crates/big-cluster/tests/config.rs b/crates/big-cluster/tests/config.rs index be959ec..935544a 100644 --- a/crates/big-cluster/tests/config.rs +++ b/crates/big-cluster/tests/config.rs @@ -533,3 +533,29 @@ fn a_file_with_no_cluster_id_is_still_identified_by_its_shape() { assert_eq!(a.fingerprint(), b.fingerprint(), "which node this is does not change it"); assert_ne!(a.fingerprint(), edited.fingerprint(), "a file somebody edited does"); } + +/// **Two files that disagree about who leads the schema still describe one cluster.** +/// +/// The leader used to be folded into the shape, and that was right while it was a name read +/// once from a file. It moves at runtime now - by an operator, and by the agreement itself - +/// so a deployment whose files were updated at different moments after a *correct* failover +/// would have two nodes that could no longer talk: a successful failover causing a partition, +/// which is the worst failure on offer. The map overrides the file within a heartbeat, so +/// folding it in protected nothing. +#[test] +fn who_leads_the_schema_is_not_part_of_the_shape() { + let with = |leader: &str| { + format!( + "schema_leader = \"{leader}\"\n\ + [[node]]\nname = \"a\"\naddr = \"10.0.0.1:7654\"\nshards = \"0..64\"\n\ + [[node]]\nname = \"b\"\naddr = \"10.0.0.2:7654\"\nshards = \"64..\"\n" + ) + }; + let a = ClusterFile::parse(&with("a")).unwrap().for_node(Some("a"), "").unwrap(); + let b = ClusterFile::parse(&with("b")).unwrap().for_node(Some("a"), "").unwrap(); + assert_eq!(a.fingerprint(), b.fingerprint()); + + // The key is still required, and still has to name a node that could hold it: it seeds the + // map, and a fresh cluster has nothing else to start from. + assert!(matches!(ClusterFile::parse(&with("nobody")), Err(ConfigError::UnknownLeader { .. }))); +} diff --git a/crates/big-cluster/tests/peers.rs b/crates/big-cluster/tests/peers.rs index ab9ff2a..8f6c900 100644 --- a/crates/big-cluster/tests/peers.rs +++ b/crates/big-cluster/tests/peers.rs @@ -24,6 +24,7 @@ //! a real clock. Those stay in `big-http/tests/cluster.rs`, over real ports, where they belong. use big_cluster::client::{ClientError, PeerResponse, Peers, Repeatable}; +use big_cluster::raft::Store as _; use big_cluster::{ raft, wire, Cluster, ClusterConfig, ClusterError, ClusterFile, FactValue, OwnedFact, }; @@ -282,6 +283,46 @@ fn a_node_whose_vote_is_unreadable_refuses_to_start() { assert_eq!(e.kind(), std::io::ErrorKind::InvalidData, "{e}"); } +/// **A restart replays only what was committed, counted from the base.** +/// +/// The replay compared a position in the vector holding the log's suffix against the commit +/// index, which is a log position. Once anything had been compacted away, an entry past the +/// commit point looked committed, and a map no majority ever agreed to became this node's map. +#[test] +fn a_restart_applies_nothing_past_the_commit_point_once_the_log_has_a_base() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.raft"); + let config: ClusterConfig = ClusterFile::parse(THREE).unwrap().for_node(Some("a"), "").unwrap(); + let at = |epoch: u64| { + let mut m = config.seed_map(); + m.epoch = epoch; + raft::Decision::Ranges(m) + }; + let log = vec![ + // Index 40, the sentinel; 41, committed; 42, still in flight. + raft::Entry { term: 3, decision: at(2) }, + raft::Entry { term: 4, decision: at(3) }, + raft::Entry { term: 4, decision: at(4) }, + ]; + raft::FileStore::new(&path) + .save(&raft::State { term: 4, voted_for: Some(0), commit: 41, base: 40, log, seed: vec![] }) + .unwrap(); + + let api = Api::in_memory().unwrap(); + let peers = Fake::new(vec![None, Some(Reply::Unreachable), Some(Reply::Unreachable)]); + let c = Cluster::with_peers( + api, + config, + peers, + Box::new(raft::FileStore::new(&path)), + raft::Timing::default(), + Default::default(), + ) + .unwrap(); + assert_eq!(c.map().epoch, 3, "the committed map, not the one still in flight"); + c.stop(); +} + /// The other half of the same rule: **a file that is simply not there is a fresh node**, and a /// fresh node starts. Without this the fix above would be a cluster that cannot be deployed. #[test] diff --git a/crates/big-cluster/tests/raft.rs b/crates/big-cluster/tests/raft.rs index d29dc83..2a8f054 100644 --- a/crates/big-cluster/tests/raft.rs +++ b/crates/big-cluster/tests/raft.rs @@ -24,6 +24,7 @@ //! never lead the same term, and a committed decision is never taken back.** Everything else //! is a way of stressing that. +use big_cluster::controller::{Controller, Leases}; use big_cluster::raft::{ Decision, Member, MemberState, Message, NodeId, Raft, Range, RangeMap, Role, State, Store, Timing, @@ -172,7 +173,14 @@ fn map_of(primary: &[NodeId], stale: &[NodeId]) -> RangeMap { moving: None, }) .collect(); - RangeMap { epoch: 1, ranges, stale: stale.to_vec(), schema_leader: 0 } + RangeMap { + epoch: 1, + ranges, + stale: stale.to_vec(), + schema_leader: 0, + reserved: Vec::new(), + schema_ready: true, + } } /// A node alone in its config file leads immediately: a majority of one is one. @@ -537,11 +545,18 @@ fn a_restart_keeps_the_term_the_vote_and_the_log() { ]), }, ]; - let state = State { term: 4, voted_for: Some(2), commit: 3, base: 0, log: log.clone() }; + let state = State { + term: 4, + voted_for: Some(2), + commit: 3, + base: 0, + log: log.clone(), + seed: voters(3), + }; store.save(&state).unwrap(); let back = store.load().unwrap().expect("just written"); - assert_eq!(back, state); + assert_eq!(back, state, "the seed included: a snapshot replaces it, so it has to be kept"); // **The commit point is part of the state.** A node that forgot it would replay its whole // log into the state machine on the way up, applying a map no majority ever agreed to. @@ -549,7 +564,9 @@ fn a_restart_keeps_the_term_the_vote_and_the_log() { // A node with no vote is a different state from a node that voted for node zero, and the // two must not encode alike. - store.save(&State { term: 9, voted_for: None, commit: 1, base: 0, log }).unwrap(); + store + .save(&State { term: 9, voted_for: None, commit: 1, base: 0, log, seed: Vec::new() }) + .unwrap(); assert_eq!(store.load().unwrap().unwrap().voted_for, None); } @@ -562,7 +579,9 @@ fn a_restarted_node_still_refuses_a_stale_candidate() { big_cluster::raft::Entry { term: 0, decision: Decision::Noop }, big_cluster::raft::Entry { term: 7, decision: owning(&[1, 0]) }, ]; - store.save(&State { term: 7, voted_for: Some(1), commit: 1, base: 0, log }).unwrap(); + store + .save(&State { term: 7, voted_for: Some(1), commit: 1, base: 0, log, seed: Vec::new() }) + .unwrap(); let state = store.load().unwrap().unwrap(); let mut node = Raft::new(0, voters(3), Timing::default(), 0); @@ -591,6 +610,7 @@ fn a_damaged_state_file_is_refused() { commit: 0, base: 0, log: vec![big_cluster::raft::Entry { term: 0, decision: Decision::Noop }], + seed: Vec::new(), }) .unwrap(); @@ -769,7 +789,7 @@ fn a_restarted_node_comes_back_with_the_cluster_as_the_log_left_it() { // Started from a file that knows three nodes, restored from a log that knows four. let mut node = Raft::new(0, voters(3), Timing::default(), 0); assert_eq!(node.members(), 3); - node.restore(State { term: 5, voted_for: Some(0), commit: 1, base: 0, log }); + node.restore(State { term: 5, voted_for: Some(0), commit: 1, base: 0, log, seed: Vec::new() }); assert_eq!(node.members(), 4, "the log is what says who is in the cluster"); assert_eq!(node.voters(), &[0, 1, 2], "and the fourth is still catching up"); @@ -887,12 +907,17 @@ fn a_follower_that_falls_behind_the_base_is_caught_up_by_a_snapshot() { fn a_compacted_log_survives_a_restart() { let dir = tempfile::tempdir().unwrap(); let store = big_cluster::raft::FileStore::new(dir.path().join("state.raft")); + // With a reservation in it: the ceiling on record ids is the one thing in the map that a + // successor cannot rebuild from anywhere else, so it had better survive a restart. + let mut kept = map_of(&[1, 0], &[]); + kept.reserve("tx", 1 << 16); + kept.schema_ready = false; let log = vec![ // The sentinel is the state as of the base, not an empty entry. - big_cluster::raft::Entry { term: 3, decision: Decision::Ranges(map_of(&[1, 0], &[])) }, + big_cluster::raft::Entry { term: 3, decision: Decision::Ranges(kept) }, big_cluster::raft::Entry { term: 4, decision: Decision::Noop }, ]; - let state = State { term: 4, voted_for: Some(1), commit: 41, base: 40, log }; + let state = State { term: 4, voted_for: Some(1), commit: 41, base: 40, log, seed: voters(3) }; store.save(&state).unwrap(); assert_eq!(store.load().unwrap().unwrap(), state, "the base is part of what is written down"); @@ -902,3 +927,246 @@ fn a_compacted_log_survives_a_restart() { assert_eq!(node.last_index(), 41, "indices are log positions, not vector positions"); assert_eq!(node.commit_index(), 41); } + +/// **A restart lands on the seed the snapshot handed over**, not on the file's. +/// +/// A follower that fell behind the base is given the state whole, members included, and takes +/// those members as its seed. Until the seed was written down, a restart forgot that and went +/// back to the file - which, once anybody had joined or left, named a cluster that was gone. +#[test] +fn a_restored_seed_is_the_membership_when_the_log_says_nothing() { + let mut grown = voters(3); + grown.push(learner("joined")); + let log = vec![big_cluster::raft::Entry { term: 3, decision: Decision::Noop }]; + + let mut node = Raft::new(0, voters(3), Timing::default(), 0); + node.restore(State { term: 3, voted_for: None, commit: 40, base: 40, log, seed: grown }); + assert_eq!(node.members(), 4, "the seed that was written down is the one that counts"); + assert_eq!(node.voters(), &[0, 1, 2], "and the fourth is a learner, as the seed said"); +} + +/// A state file from before the seed was recorded still loads, and reads as "the file's". +#[test] +fn a_state_file_from_before_the_seed_was_recorded_still_loads() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("state.raft"); + // `BIGRAFT2`, by hand: magic, term, vote, commit, base, then a log of one `Noop`. + let mut bytes = b"BIGRAFT2".to_vec(); + for v in [7u64, u64::MAX, 0, 0, 1, 0] { + bytes.extend_from_slice(&v.to_le_bytes()); + } + bytes.push(0); + std::fs::write(&path, &bytes).unwrap(); + + let state = big_cluster::raft::FileStore::new(&path) + .load() + .unwrap() + .expect("a file from the previous format is still a file"); + assert_eq!(state.term, 7); + assert_eq!(state.voted_for, None); + assert_eq!(state.log.len(), 1); + assert!(state.seed.is_empty(), "nothing was written, so nothing is claimed"); +} + +/// **Compaction keeps the newest map and the newest membership**, whatever else it drops. +/// +/// The follower's snapshot path folds the state into `log[0]`; the leader's own compaction did +/// not, so a leader that compacted past its last decision and then restarted came back with the +/// file's map and the file's members - a cluster that, once anything had moved or joined, no +/// longer existed. +#[test] +fn a_compacted_log_still_holds_the_map_and_the_membership() { + let mut sim = Sim::new(4); + sim.run(3_000); + let leader = sim.leader(); + // The fourth node steps back to learning: still replicated to, no longer voting. + let mut shrunk = voters(4); + shrunk[3].state = MemberState::Learner; + + assert!(sim.propose(leader, owning(&[1, 2, 0]))); + sim.run(500); + assert!(sim.propose(leader, Decision::Members(shrunk))); + sim.run(500); + for _ in 0..(Raft::KEEP_ENTRIES + 8) { + assert!(sim.propose(leader, Decision::Noop)); + sim.run(100); + } + sim.nodes[leader].compact(0); + assert!(sim.nodes[leader].base() > 0, "there was something to compact"); + + let log = sim.nodes[leader].log(); + assert!( + log.iter().any(|e| matches!(e.decision, Decision::Ranges(_))), + "the newest map is still in the log" + ); + assert!( + log.iter().any(|e| matches!(e.decision, Decision::Members(_))), + "and so is the newest membership" + ); + + // And a restart from that log lands on both, whatever the file says. + let mut back = Raft::new(leader, voters(4), Timing::default(), 0); + back.restore(sim.nodes[leader].state()); + assert_eq!(back.members(), 4); + assert_eq!(back.voters(), &[0, 1, 2], "the membership survived the compaction and the restart"); +} + +/// **A compacted leader still fails a range over.** +/// +/// The guard in front of a promotion compared the commit index - a log position - with the +/// length of the vector holding the log's suffix. Once anything had been compacted away the +/// two could never agree again, and every promotion after the first compaction was silently +/// refused on a cluster reporting itself healthy. +#[test] +fn a_compacted_leader_still_fails_a_range_over() { + let mut sim = Sim::new(3); + sim.run(3_000); + let leader = sim.leader(); + for i in 0..8 { + assert!(sim.propose(leader, owning(&[i % 3, (i + 1) % 3]))); + sim.run(200); + } + sim.nodes[leader].compact(0); + assert!(sim.nodes[leader].base() > 0, "the log was compacted"); + + // One copy of a replicated range goes quiet for longer than the lease allows. + let dead = (leader + 1) % 3; + sim.down.insert(dead); + let leases = Leases::default(); + sim.run(leases.promote_after.as_millis() as u64 * 3); + assert_eq!(sim.leader(), leader, "two of three are still a majority"); + + let mut current = map_of(&[dead], &[]); + current.ranges[0].group = vec![dead, leader]; + let next = Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now) + .expect("a range whose primary went quiet is given to the copy that is answering"); + assert_eq!(next.ranges[0].primary, leader); + assert!(next.is_stale(dead), "and the one that went quiet is marked behind"); +} + +// ------------------------------------------------------------------------------------------- +// The row-key namespace, moved by the agreement +// +// The most expensive decision the agreement makes: it copies every row key of every table to +// the successor. So it waits far longer than a range failover, it happens only when an operator +// has said it may, and the node it takes the namespace from is marked behind whether or not it +// holds a copy of anything - it may have interned row ids nobody else ever saw. +// ------------------------------------------------------------------------------------------- + +/// A cluster of `n` in which the range is served by the agreement's leader, so that nothing +/// about the *ranges* moves and what is left to observe is the namespace alone. +fn quiet_schema_leader(n: usize) -> (Sim, NodeId, NodeId) { + let mut sim = Sim::new(n); + sim.run(3_000); + let leader = sim.leader(); + let holder = (leader + 1) % n; + sim.down.insert(holder); + (sim, leader, holder) +} + +fn schema_after(ms: u64) -> Leases { + Leases { move_schema_after: Some(std::time::Duration::from_millis(ms)), ..Leases::default() } +} + +/// **Silence long enough for a range is not long enough for the namespace.** Moving it copies +/// every row key of every table, so it waits until the ranges have already failed over and the +/// map has settled - and on a blip it does nothing at all. +#[test] +fn the_namespace_waits_far_longer_than_a_range_does() { + let (mut sim, leader, holder) = quiet_schema_leader(3); + let leases = schema_after(15_000); + let mut current = map_of(&[leader], &[]); + current.schema_leader = holder; + + // Past `promote_after`, nowhere near the namespace's own wait. + sim.run(6_000); + assert!( + Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now).is_none(), + "a range would have moved by now; the namespace has not" + ); + + sim.run(12_000); + let next = Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now) + .expect("silence for the whole of the longer wait moves it"); + assert_eq!(next.schema_leader, leader); + assert!(!next.schema_ready, "the successor holds no row keys yet, and says so"); + assert!(next.is_stale(holder), "the deposed node is marked behind, holding a copy or not"); +} + +/// **Off unless an operator turned it on.** The namespace stays where the file put it, however +/// long its holder has been gone - which is what every deployment before this did. +#[test] +fn the_namespace_is_never_moved_unless_a_lease_says_it_may() { + let (mut sim, leader, holder) = quiet_schema_leader(3); + let mut current = map_of(&[leader], &[]); + current.schema_leader = holder; + sim.run(60_000); + assert_eq!( + Controller::promotion_for(&Leases::default(), &sim.nodes[leader], ¤t, sim.now), + None + ); +} + +/// The successor is a voter that is answering and has not missed a write - the same three +/// conditions a range's replacement has to meet, for the same reasons. +#[test] +fn a_learner_a_draining_node_and_a_copy_that_is_behind_are_all_passed_over() { + let mut sim = + Sim::with_timing(4, Timing { election_min: 1_000, election_spread: 1_000, heartbeat: 200 }); + sim.run(3_000); + let leader = sim.leader(); + // Everybody but the leader is unfit in some way: one is a learner, one is behind, and the + // fourth is the node being deposed. + let holder = (leader + 1) % 4; + let learner_at = (leader + 2) % 4; + let behind_at = (leader + 3) % 4; + let mut members = voters(4); + members[learner_at].state = MemberState::Learner; + assert!(sim.propose(leader, Decision::Members(members))); + sim.run(500); + + sim.down.insert(holder); + sim.run(20_000); + + let mut current = map_of(&[leader], &[behind_at]); + current.schema_leader = holder; + let next = + Controller::promotion_for(&schema_after(15_000), &sim.nodes[leader], ¤t, sim.now) + .expect("there is one node left that is fit to hold it"); + assert_eq!(next.schema_leader, leader, "not the learner, not the one marked behind"); +} + +/// Two leaders elected in sequence decide the same thing from the same facts, which is what +/// keeps the namespace from moving twice. +#[test] +fn the_same_facts_always_choose_the_same_successor() { + let (mut sim, leader, holder) = quiet_schema_leader(3); + sim.run(20_000); + let mut current = map_of(&[leader], &[]); + current.schema_leader = holder; + let leases = schema_after(15_000); + + let once = Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now); + for _ in 0..20 { + assert_eq!(Controller::promotion_for(&leases, &sim.nodes[leader], ¤t, sim.now), once); + } + assert!(once.is_some()); +} + +/// Nothing is decided while the namespace is still being handed over: the successor has not +/// finished taking it, and choosing another one would throw away the keys it already holds. +#[test] +fn a_handover_that_has_not_finished_is_not_restarted() { + let (mut sim, leader, holder) = quiet_schema_leader(3); + sim.run(20_000); + // Already deposed, already waiting on its successor. + let mut current = map_of(&[leader], &[holder]); + current.schema_leader = leader; + current.schema_ready = false; + + assert_eq!( + Controller::promotion_for(&schema_after(15_000), &sim.nodes[leader], ¤t, sim.now), + None, + "the node it names is this one, and it is answering; there is nothing to decide" + ); +} diff --git a/crates/big-cluster/tests/rangemap.rs b/crates/big-cluster/tests/rangemap.rs index f56606f..ee87241 100644 --- a/crates/big-cluster/tests/rangemap.rs +++ b/crates/big-cluster/tests/rangemap.rs @@ -21,7 +21,8 @@ use big_cluster::config::ClusterFile; use big_cluster::raft::{ - MapError, MergeError, Move, MoveError, MoveState, Range, RangeMap, SplitError, + MapError, Member, MemberState, MergeError, Move, MoveError, MoveState, Range, RangeMap, + SplitError, }; use big_engine::ShardRange; use proptest::prelude::*; @@ -31,7 +32,69 @@ fn range(id: u64, start: u64, end: Option, primary: usize) -> Range { } fn map(ranges: Vec) -> RangeMap { - RangeMap { epoch: 0, ranges, stale: Vec::new(), schema_leader: 0 } + RangeMap { + epoch: 0, + ranges, + stale: Vec::new(), + schema_leader: 0, + reserved: Vec::new(), + schema_ready: true, + } +} + +// ------------------------------------------------------------------------------------------- +// What the schema leader may hand out +// ------------------------------------------------------------------------------------------- + +/// **A reservation only ever rises**, and a table nothing was reserved for is at zero. Rising +/// only is what makes a reservation safe to apply twice and a block nobody used harmless: it +/// is a promise about ids that *may* have been handed out, and the higher promise keeps both. +#[test] +fn a_reservation_only_ever_rises_and_stays_sorted_by_table() { + let mut m = map(vec![range(0, 0, None, 0)]); + assert_eq!(m.reserved_for("tx"), 0, "nothing reserved is a ceiling of nothing"); + + m.reserve("tx", 65_536); + m.reserve("aa", 10); + m.reserve("tx", 100); + assert_eq!(m.reserved_for("tx"), 65_536, "a lower reservation changes nothing"); + assert_eq!(m.reserved_for("aa"), 10); + let tables: Vec<&str> = m.reserved.iter().map(|(t, _)| t.as_str()).collect(); + assert_eq!(tables, ["aa", "tx"], "sorted, so two leaders build the same bytes"); + + m.reserve("tx", 131_072); + assert_eq!(m.reserved_for("tx"), 131_072); +} + +/// **The schema leader has to be a node that can hold the namespace.** The map alone cannot +/// tell - it holds an index into a membership it does not carry - so every proposal is checked +/// against both, and a map naming a node that left or one still catching up is refused before +/// it can commit and leave nobody assigning row ids. +#[test] +fn a_schema_leader_that_left_or_is_still_learning_is_refused() { + let member = |name: &str, state| Member { + name: name.to_string(), + addr: format!("10.0.0.1:{}", name.len()), + state, + }; + let mut m = map(vec![range(0, 0, None, 0)]); + let members = vec![ + member("a", MemberState::Voter), + member("b", MemberState::Learner), + member("c", MemberState::Gone), + ]; + assert_eq!(m.check_with(&members), Ok(()), "a voter may lead"); + + m.schema_leader = 1; + assert_eq!(m.check_with(&members), Err(MapError::SchemaLeaderIsALearner { node: 1 })); + m.schema_leader = 2; + assert_eq!(m.check_with(&members), Err(MapError::SchemaLeaderNotAMember { node: 2 })); + m.schema_leader = 7; + assert_eq!(m.check_with(&members), Err(MapError::SchemaLeaderNotAMember { node: 7 })); + + // And the space is still checked first: a gap is a gap whoever leads. + let gappy = map(vec![range(0, 0, Some(8), 0), range(1, 16, None, 0)]); + assert_eq!(gappy.check_with(&members), Err(MapError::Gap { from: 8, to: 16 })); } // ------------------------------------------------------------------------------------------- diff --git a/crates/big-cluster/tests/wire.rs b/crates/big-cluster/tests/wire.rs index c87668a..0286aa5 100644 --- a/crates/big-cluster/tests/wire.rs +++ b/crates/big-cluster/tests/wire.rs @@ -402,6 +402,22 @@ proptest! { } } +/// A request to the schema leader carries who the sender thinks leads, so that the receiver can +/// disagree - and a caller with no assumption says so rather than inventing one. +#[test] +fn a_request_to_the_schema_leader_carries_who_it_thinks_leads() { + let intern = wire::InternRequest { + table: "t".to_string(), + field: "f".to_string(), + keys: vec!["GB".to_string(), "FR".to_string()], + led: Some(wire::Led { epoch: 9, leader: 2 }), + }; + assert_eq!(wire::InternRequest::decode(&intern.encode()).unwrap(), intern); + + let allocate = wire::AllocateRequest { table: "t".to_string(), count: 4, led: None }; + assert_eq!(wire::AllocateRequest::decode(&allocate.encode()).unwrap(), allocate); +} + /// A segment travels as what its records *hold*, not as encoded blocks. /// /// Shipping the encoded form would tie two nodes to the same codec choice for ever; shipping the diff --git a/crates/big-db/src/db/mod.rs b/crates/big-db/src/db/mod.rs index bd066d3..a948883 100644 --- a/crates/big-db/src/db/mod.rs +++ b/crates/big-db/src/db/mod.rs @@ -74,6 +74,11 @@ impl Db

{ Ok(self.store.set_durability(next)?) } + /// Gives trailing free pages back to the filesystem. See [`Store::truncate_tail`]. + pub fn reclaim(&self) -> Result { + Ok(self.store.truncate_tail()?) + } + pub fn store(&self) -> &Store

{ &self.store } diff --git a/crates/big-embed/src/lib.rs b/crates/big-embed/src/lib.rs index 5660ef3..a592804 100644 --- a/crates/big-embed/src/lib.rs +++ b/crates/big-embed/src/lib.rs @@ -1416,6 +1416,26 @@ impl Api

{ self.db.store().metrics() } + /// Gives trailing free pages back to the filesystem, and answers with how many. + /// + /// **The only thing in the tree that makes a served database smaller.** Copy-on-write + /// leaves holes and the freelist reuses them, so a file that has churned is mostly free + /// space that is never handed back; `big compact` gives it back and wants the exclusive + /// lock, which means stopping the daemon. This does it in place, while serving. + /// + /// **Only pages flush against the end of the file**, because moving a page anywhere else + /// means rewriting whoever points at it. What makes the tail empty out on its own is that + /// the freelist hands out its lowest page first: ordinary writes fill the holes near the + /// front, and the end of the file drains. + /// + /// [`ApiError`] `ReadersActive` when a read transaction is open anywhere in this process - + /// shrinking the file turns the region past the new end back into unbacked mapping, and a + /// borrow into it would be a `SIGBUS`. It is an ordinary outcome rather than a failure: + /// try again when the node is between requests. + pub fn reclaim(&self) -> Result { + Ok(self.db.reclaim()?) + } + /// What the row-key dictionary costs this process. See [`Db::key_stats`]. pub fn key_stats(&self) -> KeyStats { self.db.key_stats() diff --git a/crates/big-http/src/lib.rs b/crates/big-http/src/lib.rs index d56dcbf..ab73eb6 100644 --- a/crates/big-http/src/lib.rs +++ b/crates/big-http/src/lib.rs @@ -47,6 +47,7 @@ pub mod json; pub mod metrics; pub mod routes; pub mod status; +mod steward; mod watchdog; /// Re-exported from [`big_wire`], which is where the HTTP/1.1 framing lives now. @@ -112,6 +113,13 @@ pub struct ServerConfig { /// /// When the cluster may reshape itself, and how hard. Off by default. pub balance: big_cluster::balance::Policy, + /// Whether this node may give trailing free pages back to the filesystem while it serves. + /// + /// Off by default, like everything else that acts without being asked. What it changes is + /// that a file which has churned can get smaller without stopping the daemon; what it + /// costs is the write lock for the length of one truncation, and only once a quarter of + /// the file is reclaimable. + pub reclaim: bool, /// An ordinary `Option` with no `#[cfg]` on it: `TlsConfig` is uninhabited in a build with /// the feature off, so this is provably `None` there and every construction site in the tree /// - tests included - compiles either way without knowing which build it is in. @@ -145,6 +153,7 @@ impl Default for ServerConfig { // server is the default, and `big serve` refuses to bind anywhere else without one. tls: None, balance: big_cluster::balance::Policy::default(), + reclaim: false, // Off. A daemon that backed itself up somewhere by default would be a daemon // filling a disk nobody chose. backup_dir: None, @@ -316,6 +325,18 @@ impl Server

{ }) }; + // The node's own slow work - balancing, and whatever else is switched on - on a thread + // that is neither the agreement's nor a worker's. See `steward.rs` for why it can be + // neither. Stopped the way the workers are: `stopping` is set on every way out of this + // loop, and it is joined after them. + let steward = { + let state = Arc::clone(&self.state); + std::thread::Builder::new() + .name("big-steward".to_string()) + .spawn(move || steward::run(&state)) + .expect("one steward thread") + }; + let mut workers = Vec::with_capacity(self.state.config.workers); for _ in 0..self.state.config.workers { let rx = Arc::clone(&rx); @@ -397,12 +418,17 @@ impl Server

{ // peer it has given up waiting for. Err(e) if transient(&e) => continue, Err(e) => { + // A listener that has failed is a server that is stopping, whatever the + // caller asked for; the steward has to hear that too, or it outlives the + // port it was serving. + self.state.stopping.store(true, Ordering::Relaxed); drop(tx); drop(shed_tx); for w in workers { let _ = w.join(); } let _ = shedder.join(); + let _ = steward.join(); return Err(e); } } @@ -414,6 +440,7 @@ impl Server

{ let _ = w.join(); } let _ = shedder.join(); + let _ = steward.join(); Ok(()) } diff --git a/crates/big-http/src/metrics.rs b/crates/big-http/src/metrics.rs index 055ac1b..5d6281d 100644 --- a/crates/big-http/src/metrics.rs +++ b/crates/big-http/src/metrics.rs @@ -62,6 +62,11 @@ pub struct ServerMetrics { /// clients that got a connection reset instead of a `503`, which is a different support /// call from the one a `503` produces. connections_shed_unspoken: AtomicU64, + /// Pages the steward has given back to the filesystem, and the passes on which it could + /// not because a query was in flight. The pair is the whole story: a node whose file never + /// shrinks and whose blocked count climbs is a node that is never idle for long enough. + pages_reclaimed: AtomicU64, + reclaim_blocked: AtomicU64, /// Handshakes that never completed: a wrong CA, an expired certificate, a client that hung /// up, or plaintext sent to a TLS port. tls_handshakes_failed: AtomicU64, @@ -92,6 +97,17 @@ impl ServerMetrics { self.connections_rejected.fetch_add(1, Ordering::Relaxed); } + /// Trailing free pages handed back to the filesystem. + pub fn pages_reclaimed(&self, pages: u64) { + self.pages_reclaimed.fetch_add(pages, Ordering::Relaxed); + } + + /// A pass that found something to give back and could not, because a read transaction was + /// open. An ordinary outcome on a busy node, and only worth looking at as a rate. + pub fn reclaim_blocked(&self) { + self.reclaim_blocked.fetch_add(1, Ordering::Relaxed); + } + /// A connection closed without being told it was being shed, because the thread that says so /// was itself saturated. pub fn connection_shed_unspoken(&self) { @@ -166,6 +182,21 @@ impl ServerMetrics { "Connections closed without a 503, because the thread that answers one was busy.", g(&self.connections_shed_unspoken), ); + counter( + &mut out, + "big_pages_reclaimed_total", + "Trailing free pages given back to the filesystem while serving. Zero on a node \ + started without --reclaim, where only an offline `big compact` shrinks the file.", + g(&self.pages_reclaimed), + ); + counter( + &mut out, + "big_reclaim_blocked_total", + "Passes that found pages to give back and could not, because a query was in \ + flight. Climbing while big_pages_reclaimed_total stays flat is a node that is \ + never idle long enough to shrink.", + g(&self.reclaim_blocked), + ); counter( &mut out, "big_http_tls_handshakes_failed_total", @@ -447,7 +478,7 @@ pub fn render_keys(out: &mut String, k: &big_embed::KeyStats) { /// **`big_cluster_copies_behind` is the number to alert on.** It is redundancy this cluster has /// lost and will not get back until somebody runs `POST /repair`; everything else here recovers /// on its own. -pub fn render_cluster(out: &mut String, c: &big_cluster::counters::Snapshot) { +pub fn render_cluster(out: &mut String, c: &big_cluster::counters::Snapshot, balancing: bool) { gauge( out, "big_cluster_nodes", @@ -460,6 +491,81 @@ pub fn render_cluster(out: &mut String, c: &big_cluster::counters::Snapshot) { "Copies the agreement has marked behind. Redundancy lost until a repair is run.", c.behind as u64, ); + // **The second number to alert on.** A move takes seconds; a range that has been moving + // for minutes is one the balancer will plan nothing around, cluster-wide, until an + // operator runs `POST /admin/cluster/cancel`. Nothing else here says so. + gauge( + out, + "big_cluster_ranges_moving", + "Ranges with a move in flight. Nonzero for longer than a move takes means the balancer \ + is silenced until the move is cancelled or finished.", + c.moving as u64, + ); + gauge( + out, + "big_cluster_schema_ready", + "1 when the schema leader holds the row-key namespace. 0 is a handover in progress, \ + during which no new row key can be assigned.", + c.schema_ready as u64, + ); + gauge( + out, + "big_cluster_schema_handover_blocked", + "1 when handing the namespace to an elected successor found two surviving nodes \ + disagreeing about a row id. Nobody assigns row ids until an operator looks.", + c.handover_blocked as u64, + ); + gauge( + out, + "big_cluster_balancer_enabled", + "1 when this node was started with --balance and may reshape the cluster by itself.", + balancing as u64, + ); + counter( + out, + "big_cluster_peer_load_unanswered_total", + "Times a peer did not say what it weighs. To the balancer that node is neither a \ + source nor a destination, so a climbing count is a cluster that will not reshape.", + c.counts.load_unanswered, + ); + counter( + out, + "big_cluster_balance_moves_total", + "Ranges the balancer moved.", + c.counts.balance_moves, + ); + counter( + out, + "big_cluster_balance_splits_total", + "Tails the balancer cut for a node with nothing.", + c.counts.balance_splits, + ); + counter( + out, + "big_cluster_balance_admits_total", + "Learners the balancer made full members once they answered and had caught up.", + c.counts.balance_admits, + ); + counter( + out, + "big_cluster_balance_errors_total", + "Balancing steps that failed.", + c.counts.balance_errors, + ); + counter( + out, + "big_cluster_balance_drops_failed_total", + "Moves that landed but whose source could not let go of its copy. Space that is not \ + reclaimed until somebody looks.", + c.counts.balance_drops_failed, + ); + counter( + out, + "big_cluster_move_cancel_failed_total", + "Moves that failed and could not be called off either. Each one is a range still \ + marked moving.", + c.counts.move_cancel_failed, + ); gauge( out, "big_cluster_serving", diff --git a/crates/big-http/src/routes/mod.rs b/crates/big-http/src/routes/mod.rs index c784073..303427c 100644 --- a/crates/big-http/src/routes/mod.rs +++ b/crates/big-http/src/routes/mod.rs @@ -235,6 +235,8 @@ enum Target<'a> { PeerDelete, PeerIntern, PeerAllocate, + PeerSchemaStepDown, + PeerReserve, PeerNextRecord, PeerDdl, PeerDigest, @@ -297,6 +299,8 @@ impl<'a> Target<'a> { | Self::PeerDelete | Self::PeerIntern | Self::PeerAllocate + | Self::PeerSchemaStepDown + | Self::PeerReserve | Self::PeerNextRecord | Self::PeerDdl | Self::PeerRaft @@ -395,6 +399,8 @@ impl<'a> Target<'a> { | Self::PeerDelete | Self::PeerIntern | Self::PeerAllocate + | Self::PeerSchemaStepDown + | Self::PeerReserve | Self::PeerDdl | Self::PeerRaft | Self::PeerFragmentPut @@ -432,6 +438,8 @@ fn resolve<'a>(method: &str, segments: &[&'a str]) -> Option> { ("POST", ["internal", "delete"]) => Target::PeerDelete, ("POST", ["internal", "intern"]) => Target::PeerIntern, ("POST", ["internal", "allocate"]) => Target::PeerAllocate, + ("POST", ["internal", "schema", "step-down"]) => Target::PeerSchemaStepDown, + ("POST", ["internal", "reserve"]) => Target::PeerReserve, ("POST", ["internal", "next-record"]) => Target::PeerNextRecord, ("POST", ["internal", "ddl"]) => Target::PeerDdl, ("POST", ["internal", "digest"]) => Target::PeerDigest, @@ -541,7 +549,7 @@ pub fn dispatch(ctx: &Ctx<'_, P>, req: &Request) -> Answered Target::Metrics => { let mut text = ctx.metrics.render(&ctx.api().metrics()); crate::metrics::render_keys(&mut text, &ctx.api().key_stats()); - crate::metrics::render_cluster(&mut text, &ctx.cluster.counters()); + crate::metrics::render_cluster(&mut text, &ctx.cluster.counters(), ctx.balance.enabled); let (verifications, hits, throttled) = ctx.auth.counters(); crate::metrics::render_auth(&mut text, verifications, hits, throttled); Response::text( @@ -570,6 +578,8 @@ pub fn dispatch(ctx: &Ctx<'_, P>, req: &Request) -> Answered Target::PeerDelete => peer_delete(ctx, req), Target::PeerIntern => peer_intern(ctx, req), Target::PeerAllocate => peer_allocate(ctx, req), + Target::PeerSchemaStepDown => peer_schema_step_down(ctx, req), + Target::PeerReserve => peer_reserve(ctx, req), Target::PeerNextRecord => peer_next_record(ctx, req), Target::PeerDdl => peer_ddl(ctx, req), Target::PeerDigest => peer_digest(ctx, req), diff --git a/crates/big-http/src/routes/peer.rs b/crates/big-http/src/routes/peer.rs index 196f00b..9abf20c 100644 --- a/crates/big-http/src/routes/peer.rs +++ b/crates/big-http/src/routes/peer.rs @@ -125,15 +125,20 @@ pub(super) fn peer_delete(ctx: &Ctx<'_, P>, req: &Request) - /// The schema leader's one job. /// -/// Answered by whichever node receives it, which is the leader because a coordinator sends it -/// nowhere else. There is no check here that this node *is* the leader: the config is what -/// decides that, every node reads the same file, and a check would turn a misconfiguration -/// that startup already refuses into a runtime error somewhere less useful. +/// **Checked, not assumed.** It used to be assumed: the leader was a name in the config, every +/// node read the same file, and a request could only have been sent here on purpose. The +/// leader is a field in the map now, and a coordinator holding a map one decision old sends +/// this to whoever led *then* - which is how two nodes come to intern at once, the one failure +/// that hands one string two row ids and that nothing downstream can see. So the request says +/// who it thinks leads, and this node says whether it agrees before it assigns anything. pub(super) fn peer_intern(ctx: &Ctx<'_, P>, req: &Request) -> Response { let request = match wire::InternRequest::decode(&req.body) { Ok(r) => r, Err(e) => return unreadable(&e), }; + if let Err(e) = ctx.cluster.check_schema_lead(request.led.as_ref()) { + return super::from_cluster(&e); + } let keys: Vec<&str> = request.keys.iter().map(String::as_str).collect(); match ctx.api().intern_keys(&request.table, &request.field, &keys) { Ok(rows) => { @@ -154,12 +159,52 @@ pub(super) fn peer_allocate(ctx: &Ctx<'_, P>, req: &Request) Ok(r) => r, Err(e) => return unreadable(&e), }; + if let Err(e) = ctx.cluster.check_schema_lead(request.led.as_ref()) { + return super::from_cluster(&e); + } match ctx.cluster.allocate_here(&request.table, request.count) { Ok(from) => Response::binary(wire::put_u64_body(from)), Err(e) => super::from_cluster(&e), } } +/// `POST /internal/reserve`: raise the agreement's ceiling on record ids for a table. +/// +/// Asked of the agreement's leader by a schema leader that is not it, once per block of ids. +/// The body is the same shape floors travel in - `(table, one past the highest)` - and the +/// answer is the epoch the ceiling landed at. +pub(super) fn peer_reserve(ctx: &Ctx<'_, P>, req: &Request) -> Response { + let floors = match wire::get_floors(&req.body) { + Ok(f) => f, + Err(e) => return unreadable(&e), + }; + let mut epoch = 0; + for (table, upto) in &floors { + match ctx.cluster.reserve_here(table, *upto) { + Ok(at) => epoch = at, + Err(e) => return super::from_cluster(&e), + } + } + Response::binary(wire::put_u64_body(epoch)) +} + +/// `POST /internal/schema/step-down`: stop interning, the namespace is being handed over. +/// +/// The body is the map epoch the move read. This node refuses to intern or allocate until the +/// map is past it - which, once the move lands, it is, and by then the map names somebody +/// else. Answered with the epoch, so the caller knows which one was heard. +pub(super) fn peer_schema_step_down( + ctx: &Ctx<'_, P>, + req: &Request, +) -> Response { + let epoch = match wire::get_u64_body(&req.body) { + Ok(e) => e, + Err(e) => return unreadable(&e), + }; + ctx.cluster.stand_down_schema(epoch); + Response::binary(wire::put_u64_body(epoch)) +} + /// One past the highest record id this node holds, which is its share of the answer the leader /// allocates above. pub(super) fn peer_next_record(ctx: &Ctx<'_, P>, req: &Request) -> Response { @@ -387,7 +432,7 @@ pub(super) fn cluster_rebalance(ctx: &Ctx<'_, P>, req: &Requ } match ctx.cluster.rebalance(&policy) { Ok(None) => Response::ok("{\"did\":null}".to_string()), - Ok(Some(what)) => Response::ok(format!("{{\"did\":{}}}", json::string(&what))), + Ok(Some(what)) => Response::ok(format!("{{\"did\":{}}}", json::string(&what.describe()))), Err(e) => from_cluster(&e), } } diff --git a/crates/big-http/src/steward.rs b/crates/big-http/src/steward.rs new file mode 100644 index 0000000..6e5b3b3 --- /dev/null +++ b/crates/big-http/src/steward.rs @@ -0,0 +1,340 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The thread that does a node's slow, optional work on its own behalf. +//! +//! **Not the agreement's thread.** `Controller::run` ticks every fifty milliseconds with the +//! raft lock held, and everything it does must be over before the next heartbeat is due. A +//! balancing step is the opposite kind of work: it copies a range over the network, proposes to +//! the agreement and waits for the proposal to land - seconds, with the raft lock taken and +//! released along the way. Run from inside the driver it would deadlock against itself; run +//! from a worker it would hold a request's thread for the life of a move. So it has a thread of +//! its own, owned by the server, and the server stops it the way it stops its workers. +//! +//! **Every job here is off until something turns it on.** A node that reshapes its cluster, or +//! its file, without being asked is a node whose behaviour an operator cannot predict. The +//! jobs read their switches from the configuration on every pass, do nothing while the switch +//! is off, and cost one clock read a second for the privilege. + +use crate::State; +use big_cluster::raft::{MoveState, RangeId}; +use big_pager::PagerMut; +use big_wire::log; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +/// How often the jobs are considered. +/// +/// A second, not the agreement's fifty milliseconds: a pass that gets past its gates asks every +/// member what it weighs, and a pass that acts costs seconds. Nothing here is urgent. +const PERIOD: Duration = Duration::from_secs(1); + +/// How often the sleep looks up to see whether the server is standing down. +/// +/// A stop is a stop: a thread that sleeps a whole period before noticing is a shutdown that +/// takes a second longer than it should, and a test that waits on one. +const WAKE: Duration = Duration::from_millis(50); + +/// After a job fails, how long before it is tried again - and the ceiling on that. +/// +/// A cluster that cannot balance because a peer is down is not helped by being asked again +/// every second, and the peer is not helped by a request a second from every leader it has had. +const BACKOFF_FROM: Duration = Duration::from_secs(1); +const BACKOFF_TO: Duration = Duration::from_secs(60); + +/// How long a range may sit in one state of a move before somebody is told. +/// +/// A move takes seconds. Ten minutes in the same state is not a slow move; it is a move whose +/// driver died, and a range marked moving silences the balancer cluster-wide until an operator +/// runs `POST /admin/cluster/cancel`. Said once per transition rather than every pass, and +/// never acted on: a canceller racing a slow-but-progressing move would throw away real work. +const STUCK_AFTER: Duration = Duration::from_secs(600); + +/// How much of the file has to be reclaimable before the tail is worth trying to give back. +/// +/// A quarter. Below that the holes are what copy-on-write always leaves and what the freelist +/// is there to reuse, and taking the write lock to hand back a few pages would be paying for +/// nothing. +const RECLAIM_AT_PERCENT: u64 = 25; + +/// And a floor, so that a database of a few hundred pages is left alone whatever the ratio: a +/// quarter of nothing is nothing, and an empty file has no tail worth trimming. +const RECLAIM_FLOOR_PAGES: u64 = 4_096; + +/// Runs until the server stands down. +pub(crate) fn run(state: &State

) { + let mut backoff = Backoff::default(); + let mut watch = Watch::default(); + while !state.stopping.load(Ordering::Relaxed) { + if let Some((range, in_state, for_)) = watch.observe(moving(state), Instant::now()) { + log::emit( + log::Level::Warn, + "move_stuck", + &[ + ("range", log::F::N(range)), + ("state", log::F::S(&format!("{in_state:?}"))), + ("for_secs", log::F::N(for_.as_secs())), + ], + ); + } + + // The handover first: while it is open nobody assigns row ids, and nothing the + // balancer could do is worth more than closing it. + let handed = match handover(state) { + Ok(None) => Ok(()), + Ok(Some(to)) => { + log::emit(log::Level::Info, "schema_handover", &[("to", log::F::S(&to))]); + Ok(()) + } + Err(e) => { + // A survivor that did not answer is retried; a survivor that contradicts + // another is not something a retry can fix, and it is said at the level it + // deserves. Both back off: the cluster is not helped by being asked again + // every second. + let level = if e.is_unreachable() { log::Level::Warn } else { log::Level::Error }; + log::emit(level, "schema_handover_failed", &[("error", log::F::S(&e.to_string()))]); + Err(()) + } + }; + + // Local, and gated on nothing but its own switch: it asks no peer and proposes + // nothing, so it runs on every node - clustered or not, leader or not. + reclaim(state); + + let wait = match balance(state) { + Ok(None) if handed.is_ok() => PERIOD, + Ok(None) => backoff.next(), + Ok(Some(did)) => { + log::emit(log::Level::Info, "balance_step", &[("did", log::F::S(&did.describe()))]); + PERIOD + } + Err(e) => { + let wait = backoff.next(); + log::emit( + log::Level::Warn, + "balance_error", + &[ + ("error", log::F::S(&e.to_string())), + ("retry_in_ms", log::F::N(wait.as_millis() as u64)), + ], + ); + wait + } + }; + if wait <= PERIOD { + backoff.reset(); + } + sleep_unless_stopping(state, wait); + } +} + +/// One balancing step, if this node is the one that decides and the policy allows it. +/// +/// **Only the agreement's leader**, and only when everything it has proposed has landed - the +/// same two gates `promotion` stands behind, for the same reason: a decision about a map that +/// has not settled is a decision about a state that is already gone. A node with no agreement +/// has nothing to balance. +fn balance( + state: &State

, +) -> big_cluster::Result> { + let policy = &state.config.balance; + if !policy.enabled { + return Ok(None); + } + let Some(controller) = state.cluster.controller() else { + return Ok(None); + }; + if !controller.is_leader() || !controller.settled() { + return Ok(None); + } + state.cluster.rebalance(policy) +} + +/// Finishes a handover of the row-key namespace the agreement decided, if one is open. +/// +/// **The agreement's leader, and only it.** It decided the successor, it is the one node that +/// can mark the successor ready, and it reaches every survivor. The successor itself does +/// nothing here: it is given the keys, and told when it holds them all. +fn handover( + state: &State

, +) -> big_cluster::Result> { + let Some(controller) = state.cluster.controller() else { + return Ok(None); + }; + if !controller.is_leader() || !controller.settled() { + return Ok(None); + } + state.cluster.finish_schema_handover() +} + +/// Gives trailing free pages back to the filesystem, when there are enough to be worth it. +/// +/// **The only thing that makes a served database smaller.** Copy-on-write leaves holes, the +/// freelist reuses them lowest-first so the tail drains, and this hands the drained tail back. +/// `big compact` does the same job offline and wants the exclusive lock, which means stopping +/// the node. +/// +/// Three things keep it out of the way. It does nothing until a quarter of the file is +/// reclaimable, so an ordinary database is never touched. It does nothing while a backup is +/// walking the file, because a backup holds a reader for its whole run: the horizon cannot +/// move, so there would be nothing to release and the attempt would only take the write lock. +/// And `ReadersActive` - a read transaction open anywhere in this process - is an ordinary +/// outcome rather than a failure, counted and retried on the next pass rather than logged. +fn reclaim(state: &State

) { + if !state.config.reclaim { + return; + } + if state.backing_up.load(Ordering::Relaxed) { + return; + } + let m = state.cluster.local().metrics(); + if m.page_count < RECLAIM_FLOOR_PAGES + || m.free_pages_reusable * 100 < m.page_count * RECLAIM_AT_PERCENT + { + return; + } + match state.cluster.local().reclaim() { + Ok(0) => {} + Ok(pages) => { + state.metrics.pages_reclaimed(pages); + log::emit(log::Level::Info, "reclaimed", &[("pages", log::F::N(pages))]); + } + // Not a failure: it means a query was in flight, which on a busy node is most of the + // time. The counter is what says whether "most" has become "always". + Err(_) => state.metrics.reclaim_blocked(), + } +} + +/// The one move in flight, if there is one. The map allows at most one. +fn moving(state: &State

) -> Option<(RangeId, MoveState)> { + state.cluster.map().ranges.iter().find_map(|r| r.moving.as_ref().map(|m| (r.id, m.state))) +} + +/// Sleeps for `wait`, waking early if the server stands down meanwhile. +fn sleep_unless_stopping(state: &State

, wait: Duration) { + let mut left = wait; + while !left.is_zero() && !state.stopping.load(Ordering::Relaxed) { + let step = left.min(WAKE); + std::thread::sleep(step); + left -= step; + } +} + +/// Doubling, capped, reset on success. Local to the thread: nothing about a retry belongs in +/// the map. +#[derive(Default)] +struct Backoff { + current: Option, +} + +impl Backoff { + fn next(&mut self) -> Duration { + let next = self.current.map_or(BACKOFF_FROM, |c| (c * 2).min(BACKOFF_TO)); + self.current = Some(next); + next + } + + fn reset(&mut self) { + self.current = None; + } +} + +/// Watches the one move in flight and says when it has been in one state for too long. +/// +/// **Local, and deliberately so.** The obvious place for "since when" is the map, and it would +/// be wrong there twice: the balancer's determinism rests on reading no clock, and a timestamp +/// in a replicated value is a clock every node reads differently. So the steward remembers +/// what it saw and when, and forgets both the moment the move changes state or ends. +#[derive(Default)] +struct Watch { + seen: Option<(RangeId, MoveState, Instant)>, + warned: bool, +} + +impl Watch { + /// Feeds one observation. `Some` exactly once per stuck state: the moment it has lasted + /// past the threshold, and not again until the move changes. + fn observe( + &mut self, + now_moving: Option<(RangeId, MoveState)>, + now: Instant, + ) -> Option<(RangeId, MoveState, Duration)> { + match (now_moving, self.seen) { + (None, _) => { + self.seen = None; + self.warned = false; + None + } + (Some((id, st)), Some((seen_id, seen_st, since))) if seen_id == id && seen_st == st => { + let for_ = now.saturating_duration_since(since); + if for_ >= STUCK_AFTER && !self.warned { + self.warned = true; + Some((id, st, for_)) + } else { + None + } + } + (Some((id, st)), _) => { + self.seen = Some((id, st, now)); + self.warned = false; + None + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn backoff_doubles_to_a_ceiling_and_resets() { + let mut b = Backoff::default(); + assert_eq!(b.next(), Duration::from_secs(1)); + assert_eq!(b.next(), Duration::from_secs(2)); + assert_eq!(b.next(), Duration::from_secs(4)); + for _ in 0..10 { + b.next(); + } + assert_eq!(b.next(), BACKOFF_TO, "capped, however many times it fails"); + b.reset(); + assert_eq!(b.next(), Duration::from_secs(1), "a success starts the clock again"); + } + + /// A move that progresses is never reported; one that does not is reported once, and + /// again only if it stalls afresh in another state. + #[test] + fn a_stalled_move_is_reported_once_per_state() { + let mut w = Watch::default(); + let t0 = Instant::now(); + let seeding = Some((7, MoveState::Seeding)); + let cutover = Some((7, MoveState::Cutover)); + + assert_eq!(w.observe(seeding, t0), None, "first sight starts the clock"); + assert_eq!(w.observe(seeding, t0 + STUCK_AFTER / 2), None, "half way is not stuck"); + let stuck = w.observe(seeding, t0 + STUCK_AFTER); + assert_eq!(stuck.map(|(id, st, _)| (id, st)), Some((7, MoveState::Seeding))); + assert_eq!(w.observe(seeding, t0 + STUCK_AFTER * 2), None, "said once, not every pass"); + + // It moved on. The clock restarts with the new state, and warns again only if that + // one stalls too. + assert_eq!(w.observe(cutover, t0 + STUCK_AFTER * 2), None); + assert_eq!(w.observe(cutover, t0 + STUCK_AFTER * 2 + STUCK_AFTER / 2), None); + assert!(w.observe(cutover, t0 + STUCK_AFTER * 3).is_some()); + + // Finished: nothing to watch, nothing remembered. + assert_eq!(w.observe(None, t0 + STUCK_AFTER * 4), None); + assert_eq!(w.observe(seeding, t0 + STUCK_AFTER * 4), None, "a new move starts fresh"); + } +} diff --git a/crates/big-http/tests/cluster.rs b/crates/big-http/tests/cluster.rs index a4162b6..3f5790f 100644 --- a/crates/big-http/tests/cluster.rs +++ b/crates/big-http/tests/cluster.rs @@ -87,6 +87,14 @@ fn a_replicated_group() -> (SocketAddr, SocketAddr, SocketAddr) { [[node]]\nname = \"a-third\"\naddr = \"{third}\"\nreplica = \"a\"\n" ); start(&file, &[("a", a), ("a-spare", spare), ("a-third", third)]); + // **Wait for the agreement before asking anything of it.** Every node here holds a + // replicated range, so every one of them is fenced: until it has heard from the agreement + // it cannot know it has not already been replaced, and it refuses rather than risk being + // the second node answering for one range. On the shipped clocks that is an election, and + // an election is what a replicated cluster needs before it can serve at all. + waiting("an elected leader", std::time::Duration::from_secs(15), || { + ready(a).contains(r#""leader":""#) + }); (a, spare, third) } @@ -757,6 +765,9 @@ fn a_write_that_misses_a_copy_stands_and_says_so() { // `a-third` is configured and never started, so a majority is still two of three and the // agreement works - it is one copy that is missing, not the quorum. start(&file, &[("a", a), ("a-spare", spare)]); + waiting("an elected leader", std::time::Duration::from_secs(15), || { + ready(a).contains(r#""leader":""#) + }); // A schema change reaches the copies it can and names the one it cannot. let (status, body) = send(a, "POST", "/table/tx", ""); @@ -859,6 +870,7 @@ fn brisk() -> (Timing, Leases) { Leases { serve_for: std::time::Duration::from_millis(300), promote_after: std::time::Duration::from_millis(900), + ..Leases::default() }, ) } @@ -876,6 +888,17 @@ fn a_group_of_three() -> (String, Vec) { (file, ports) } +/// The same as `brisk`, and the agreement is also allowed to move the row-key namespace. +/// +/// Fifteen seconds in a deployment, a quarter of a second here, and for the same reason the +/// other clocks are shortened: what these tests are about is the rule, and the rule does not +/// depend on the number. The ordering does, and it is kept - the namespace still waits longer +/// than a range does. +fn brisk_electing() -> (Timing, Leases) { + let (timing, leases) = brisk(); + (timing, Leases { move_schema_after: Some(leases.promote_after * 3), ..leases }) +} + /// A node a test can take away. /// /// `close` drops the listener rather than only stopping the loop: a port that accepts and never @@ -930,6 +953,32 @@ fn start_with( timing: Timing, leases: Leases, state: Option, +) -> Node { + start_configured(file, name, addr, timing, leases, state, ServerConfig::default()) +} + +/// On the brisk clocks, with the balancer switched on - the one thing the steward acts on. +fn start_balancing(file: &str, name: &str, addr: SocketAddr) -> Node { + let (timing, leases) = brisk(); + let config = ServerConfig { + balance: big_cluster::balance::Policy { + enabled: true, + ..big_cluster::balance::Policy::default() + }, + ..ServerConfig::default() + }; + start_configured(file, name, addr, timing, leases, None, config) +} + +/// The same, with the server's configuration spelled out as well. +fn start_configured( + file: &str, + name: &str, + addr: SocketAddr, + timing: Timing, + leases: Leases, + state: Option, + server: ServerConfig, ) -> Node { let config = ClusterFile::parse(file).unwrap().for_node(Some(name), "").unwrap(); let store: Box = match state { @@ -939,8 +988,7 @@ fn start_with( let cluster = Cluster::with_timing(Api::in_memory().unwrap(), config, None, store, timing, leases) .unwrap(); - let server = - Server::bind_cluster(cluster, addr, ServerConfig::default()).expect("the port was free"); + let server = Server::bind_cluster(cluster, addr, server).expect("the port was free"); let running = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); let flag = std::sync::Arc::clone(&running); let done = std::thread::spawn(move || { @@ -1009,6 +1057,49 @@ fn a_range_fails_over_when_its_primary_dies() { }); } +/// **A range still fails over after the leader has compacted its log.** +/// +/// The guard in front of a promotion compared the commit index with the length of the vector +/// holding the log's suffix, and the two could never agree again once anything had been +/// compacted away - so every promotion after the first compaction was refused, silently, on a +/// cluster reporting itself healthy. The rule is proven against a simulated agreement in +/// `big-cluster/tests/raft.rs`; this is the one test that runs it inside the real driver +/// thread, against a log the real leader compacted on its own. +#[test] +fn a_range_fails_over_after_the_log_has_been_compacted() { + let (file, ports) = a_group_of_three(); + // **The copies elect a leader before the primary arrives.** Only the leader compacts its + // log - a follower's base moves only when it is handed a snapshot - so the node that dies + // must not be the one that decides, or its successor would decide from an uncompacted log + // and this test would pass with the bug in place. `a` is the primary by the file; the + // agreement's leader has to be one of the other two. + let _b = start_agreeing(&file, "b", ports[1]); + let _c = start_agreeing(&file, "c", ports[2]); + until("a leader among the copies", || ready(ports[1]).contains(r#""leader":""#)); + let mut a = start_agreeing(&file, "a", ports[0]); + until("the primary to join", || ready(ports[0]).contains(r#""leader":""#)); + let leader = leader_of(&ports); + assert_ne!(leader, ports[0], "the primary joined a settled agreement; it must not lead it"); + + ok(ports[0], "POST", "/table/tx", ""); + ok(ports[0], "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(ports[0], "POST", "/table/tx/import", "amount 1 5\namount 2 5\n"); + + // Enough committed decisions that the leader compacts. Every split and every merge is one + // entry, and with every voter up the leader drops whatever is more than `KEEP_ENTRIES` + // behind - so more than that many, with room, is a log that has certainly been cut. + for _ in 0..(big_cluster::raft::Raft::KEEP_ENTRIES / 2 + 8) { + ok(leader, "POST", "/admin/cluster/split?at=900", ""); + ok(leader, "POST", "/admin/cluster/merge?range=0", ""); + } + + a.close(); + until("the range to move after a compaction", || { + let (status, body) = send(ports[1], "POST", "/table/tx/query", "Count(All())"); + status == 200 && body == r#"{"count":2}"# + }); +} + /// **A node cut off from the agreement stops answering for its range.** /// /// This is the half that makes the other half safe. A failure detector alone cannot tell a @@ -1193,6 +1284,8 @@ fn a_second_batch_of_keys_does_not_collide_with_the_first() { let mut a_spare = start_agreeing(&file, "a-spare", spare); start_agreeing(&file, "a", a).forget(); start_agreeing(&file, "b", b).forget(); + // `a` holds a replicated range, so it is fenced until it has heard from the agreement. + until("an elected leader", || ready(a).contains(r#""leader":""#)); ok(a, "POST", "/table/tx", ""); ok(a, "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); @@ -1371,7 +1464,7 @@ fn a_node_that_restarts_keeps_what_it_agreed() { // The file was read rather than started fresh: the state on disk says so. let written = std::fs::read(dir.path().join("c.raft")).unwrap(); - assert!(written.starts_with(b"BIGRAFT2"), "the agreement wrote no state for `c`"); + assert!(written.starts_with(b"BIGRAFT3"), "the agreement wrote no state for `c`"); // Reads never stopped: the copy serving the range was never the one taken away. assert_eq!(ok(ports[0], "POST", "/table/tx/query", "Count(All())"), r#"{"count":1}"#); @@ -1415,6 +1508,7 @@ fn a_serving_node_that_comes_back_empty_is_only_found_by_verify() { let leases = Leases { serve_for: std::time::Duration::from_secs(5), promote_after: std::time::Duration::from_secs(30), + ..Leases::default() }; let mut a = start_with(&file, "a", ports[0], timing, leases, None); let _b = start_with(&file, "b", ports[1], timing, leases, None); @@ -1726,6 +1820,37 @@ fn a_node_joins_as_a_learner_and_is_promoted_when_it_has_something_to_serve() { }); } +/// **A node that joined is admitted by the steward**, once it is answering for data and holds +/// the log - and by nothing else. The test above promotes by hand; this one starts the fourth +/// node for real, with a file that names the whole cluster and itself, and waits for the +/// agreement's leader to decide it counts. +#[test] +fn a_learner_that_has_caught_up_is_admitted_by_the_steward() { + let (a, b, spare, d) = (free_port(), free_port(), free_port(), free_port()); + // Named, so that a node whose file describes four nodes is recognised by three whose files + // describe three: without a `cluster_id` the shape of the file is what nodes check. + let file = format!("cluster_id = \"grows\"\n{}", three(a, b, spare)); + let _a = start_balancing(&file, "a", a); + let _b = start_balancing(&file, "b", b); + let _spare = start_balancing(&file, "a-spare", spare); + until("an elected leader", || ready(a).contains(r#""leader":""#)); + + let leader = leader_of(&[a, b, spare]); + ok(leader, "POST", &format!("/admin/cluster/node?name=d&addr={d}"), ""); + + // The node itself, seeded as a copy of `a` - the shape that parses without overlapping + // anybody's range, and one the agreement overwrites the moment it is heard from. + let grown = format!("{file}[[node]]\nname = \"d\"\naddr = \"{d}\"\nreplica = \"a\"\n"); + let _d = start_balancing(&grown, "d", d); + + until("the steward to admit it", || { + let seen = ok(a, "GET", "/cluster/topology", ""); + seen.contains(r#""name":"d","addr""#) && !seen.contains(r#""state":"learner""#) + }); + let metrics = ok(leader, "GET", "/metrics", ""); + assert!(metrics.contains("big_cluster_balance_admits_total 1"), "{metrics}"); +} + /// **A node that still holds a range is not removed**, because removing it removes the only /// copy of what it holds and leaves the map naming a node nobody talks to. #[test] @@ -1945,6 +2070,55 @@ fn a_cancelled_move_leaves_the_range_where_it_was() { assert!(after.contains(r#""shards":"64..","primary":"b""#), "{after}"); } +/// **Switched on, the balancer needs no operator.** The same step the test below asks for by +/// hand, taken by the steward on the agreement's leader - and only there - because the policy +/// says it may. Draining `b` is the only instruction anybody gives. +#[test] +fn the_balancer_runs_by_itself_when_switched_on() { + let (a, b, spare) = (free_port(), free_port(), free_port()); + let file = three(a, b, spare); + let _a = start_balancing(&file, "a", a); + let _b = start_balancing(&file, "b", b); + let _spare = start_balancing(&file, "a-spare", spare); + until("an elected leader", || ready(a).contains(r#""leader":""#)); + + ok(a, "POST", "/table/tx", ""); + ok(a, "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(a, "POST", "/table/tx/import", &format!("amount {} 5\n", 70 * (1 << 20))); + + let leader = leader_of(&[a, b, spare]); + ok(leader, "POST", "/admin/cluster/drain?name=b", ""); + until("the steward to move the range off the draining node", || { + !ok(a, "GET", "/cluster/topology", "").contains(r#""shards":"64..","primary":"b""#) + }); + // The records came with it. + assert_eq!(ok(a, "POST", "/table/tx/query", "Count(All())"), r#"{"count":1}"#); + + // **And it stops.** `b` holds nothing, so the next pass finds nothing to do, and a + // draining node is never a destination - which is exactly the state in which it can be + // removed for good. A removal refused is one the move has not finished landing. + until("the move to be over and `b` removable", || { + send(leader, "DELETE", "/admin/cluster/node?name=b", "").0 == 200 + }); +} + +/// The steward is stopped the way the workers are, and no slower. A node asked to stand down +/// comes back within a wake of the steward's sleep, not a whole period of it. +#[test] +fn a_node_stands_down_promptly_with_its_steward() { + let (file, ports) = a_group_of_three(); + let mut a = start_balancing(&file, "a", ports[0]); + until("the node to answer", || send(ports[0], "GET", "/ready", "").0 == 200); + + let started = std::time::Instant::now(); + a.close(); + assert!( + started.elapsed() < std::time::Duration::from_secs(1), + "stood down in {:?}", + started.elapsed() + ); +} + /// **The balancer, over real nodes.** It decides from what the nodes actually weigh, and one /// call does one thing - so a cluster that needs several steps takes several calls, each /// against facts gathered afresh. @@ -1985,11 +2159,43 @@ fn rebalancing_takes_a_range_off_a_draining_node_and_then_stops() { let again = ok(leader, "POST", "/admin/cluster/rebalance?force=true", ""); assert_eq!(again, r#"{"did":null}"#); + // The step was counted where an operator scrapes, and the move is over rather than + // lingering as a range marked moving - the state that silences every step after it. + let metrics = ok(leader, "GET", "/metrics", ""); + assert!(metrics.contains("big_cluster_balance_moves_total 1"), "{metrics}"); + assert!(metrics.contains("big_cluster_ranges_moving 0"), "{metrics}"); + assert!(metrics.contains("big_cluster_balance_drops_failed_total 0"), "{metrics}"); + // Which is exactly the state in which it can be removed for good. let (status, said) = send(leader, "DELETE", "/admin/cluster/node?name=b", ""); assert_eq!(status, 200, "{said}"); } +/// **A peer that does not say what it weighs is counted, not just skipped.** To the balancer +/// that node is neither a source nor a destination, so a cluster with one silent member is a +/// cluster that quietly stops reshaping - and the only way an operator tells that apart from a +/// cluster with nothing to do is this number. +#[test] +fn a_peer_that_does_not_answer_the_balancer_is_counted() { + let (a, b, spare) = (free_port(), free_port(), free_port()); + let file = three(a, b, spare); + let mut nodes = [ + start_agreeing(&file, "a", a), + start_agreeing(&file, "b", b), + start_agreeing(&file, "a-spare", spare), + ]; + until("an elected leader", || ready(a).contains(r#""leader":""#)); + let leader = leader_of(&[a, b, spare]); + + // Take away a node that is not deciding, so the one that is keeps deciding. + let silent = [a, b, spare].into_iter().position(|p| p != leader).expect("two others"); + nodes[silent].close(); + + ok(leader, "POST", "/admin/cluster/rebalance?force=true", ""); + let metrics = ok(leader, "GET", "/metrics", ""); + assert!(metrics.contains("big_cluster_peer_load_unanswered_total 1"), "{metrics}"); +} + /// Off unless asked. A cluster that reshapes itself unasked is one whose shape an operator /// cannot predict, so the policy ships disabled and `?force=true` is what an operator uses. #[test] @@ -2009,6 +2215,182 @@ fn the_balancer_does_nothing_until_it_is_switched_on() { assert_eq!(ok(leader, "POST", "/admin/cluster/rebalance", ""), r#"{"did":null}"#); } +/// **The agreement gives the row-key namespace away when its holder dies, and nothing is +/// handed the same meaning twice.** +/// +/// Four claims, and the middle two are what make the first one safe. The namespace moves. In +/// the gap before the successor holds every key, a write with a *new* key is refused rather +/// than given an id somebody else might also be giving out. Once it is ready, a key that +/// already had a row id has the *same* one - which is the proof that the keys came across from +/// the survivors rather than being invented again. And no record id is reused across the whole +/// thing. +#[test] +fn the_agreement_moves_the_namespace_when_its_holder_dies() { + let (a, b, spare) = (free_port(), free_port(), free_port()); + let file = three(a, b, spare); + let (timing, leases) = brisk_electing(); + let mut a_node = start_with(&file, "a", a, timing, leases, None); + let _b = start_with(&file, "b", b, timing, leases, None); + let _spare = start_with(&file, "a-spare", spare, timing, leases, None); + until("an elected leader", || ready(b).contains(r#""leader":""#)); + + ok(a, "POST", "/table/tx", ""); + ok(a, "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(a, "POST", "/table/tx/field/country?kind=set", ""); + ok(a, "POST", "/sql", "INSERT INTO tx (amount, country) VALUES (5, 'GB')"); + ok(a, "POST", "/sql", "INSERT INTO tx (amount, country) VALUES (7, 'FR')"); + + // What `GB` means before the handover. It has to mean the same afterwards: a successor + // that invented a second row id for it would answer a `GroupBy` with two groups that are + // one group, and nothing downstream could tell. + let before = ok(b, "POST", "/table/tx/query", "GroupBy(All(), field=\"country\")"); + assert!(before.contains("GB") && before.contains("FR"), "{before}"); + + // The schema leader dies. Nobody is told; nothing is edited. + a_node.close(); + + // 1. It moves, and to a node that is answering. + until("the namespace to move", || { + let seen = ok(b, "GET", "/cluster/topology", ""); + seen.contains(r#""schema_leader":"b""#) || seen.contains(r#""schema_leader":"a-spare""#) + }); + + // 2. Once the successor holds the keys, an existing key resolves to the id it always had. + until("the successor to finish taking it over", || { + send(b, "POST", "/sql", "INSERT INTO tx (amount, country) VALUES (9, 'GB')").0 == 200 + }); + let after = ok(b, "POST", "/table/tx/query", "GroupBy(All(), field=\"country\")"); + assert_eq!( + after.matches("GB").count(), + 1, + "one group for `GB`, not two - the keys came across: {after}" + ); + + // 3. A key nobody has ever seen works too, now that somebody holds the namespace. + ok(b, "POST", "/sql", "INSERT INTO tx (amount, country) VALUES (11, 'DE')"); + + // 4. No record id was handed out twice: four inserts, four records. + until("both survivors to agree on the count", || { + [b, spare] + .iter() + .all(|p| send(*p, "POST", "/table/tx/query", "Count(All())").1 == r#"{"count":4}"#) + }); + + // And the node that was deposed is marked behind: it may hold row ids it interned for + // writes that never landed anywhere, so it must be repaired before it is trusted again. + assert!(ready(b).contains(r#""behind":["a"]"#), "{}", ready(b)); +} + +/// **A record id promised by a leader that died is never promised again.** +/// +/// The floor a leader allocates from lives in its memory, and a successor - here the same node, +/// restarted with an empty database - would start from what is on disk and hand out ids the +/// old leader had already given to writes that may not have landed. So the leader commits a +/// ceiling through the agreement before it hands out anything under it, in blocks, and a +/// fresh leader starts at the ceiling. The tell is the id: it jumps to the block boundary +/// rather than continuing from the highest record anybody holds. +#[test] +fn a_restarted_schema_leader_starts_above_everything_it_may_have_promised() { + let dir = tempfile::tempdir().unwrap(); + let (file, ports) = a_group_of_three(); + let state = |name: &str| Some(dir.path().join(format!("{name}.raft"))); + let (timing, leases) = brisk(); + + let mut a = start_with(&file, "a", ports[0], timing, leases, state("a")); + let _b = start_with(&file, "b", ports[1], timing, leases, state("b")); + let _c = start_with(&file, "c", ports[2], timing, leases, state("c")); + until("an elected leader", || ready(ports[1]).contains(r#""leader":""#)); + + ok(ports[1], "POST", "/table/tx", ""); + ok(ports[1], "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(ports[1], "POST", "/sql", "INSERT INTO tx (amount) VALUES (5)"); + ok(ports[1], "POST", "/sql", "INSERT INTO tx (amount) VALUES (7)"); + assert_eq!( + ok(ports[1], "GET", "/table/tx/records?limit=5", ""), + r#"{"records":[0,1],"next":null}"#, + "two ids, from zero, as always" + ); + + // The schema leader goes away and comes back with its agreement state and nothing else - + // a replaced disk. Its floor was in memory; the ceiling it committed was not. + a.close(); + until("the copy that went away to be marked behind", || { + ready(ports[1]).contains(r#""behind":["a"]"#) + }); + let _a_again = start_with(&file, "a", ports[0], timing, leases, state("a")); + until("the restarted node to answer", || send(ports[0], "GET", "/ready", "").0 == 200); + // Caught up first: it is still a copy of the range every write goes to, and a copy with + // no table in it refuses the write. That is the repair's job, not this test's subject. + until("the restarted copy to be repaired", || { + send(ports[1], "POST", "/repair", "").1.contains(r#""outcome":"caught up""#) + }); + until("the leader to allocate again", || { + send(ports[1], "POST", "/sql", "INSERT INTO tx (amount) VALUES (9)").0 == 200 + }); + + // Not 2. The highest record anybody holds is 1, and a leader starting from there would be + // free to re-issue whatever the old one promised in between. + let ids = ok(ports[1], "GET", "/table/tx/records?after=1&limit=5", ""); + assert!(ids.contains("65536"), "the next id starts at the committed block: {ids}"); +} + +/// **A schema leader that loses the agreement stops assigning row ids**, even when the range +/// it holds is not fenced. A range with no copy is never fenced because nothing could take it; +/// the namespace can always be given to somebody else, so the node that holds it acts only +/// while it can prove it still does. What it costs is exactly one thing: a write with a key +/// nobody has seen waits. A write whose keys are known lands as it always did. +#[test] +fn the_schema_leader_stops_interning_when_it_loses_the_agreement() { + let (a, b, spare) = (free_port(), free_port(), free_port()); + // `a` leads the schema and holds a range of one copy; the copy that makes the agreement + // run at all is of `b`'s range. So when `b` and its copy go, `a` keeps serving its own + // range - and cannot know whether the namespace is still its. + let file = format!( + "schema_leader = \"a\"\n\ + [[node]]\nname = \"a\"\naddr = \"{a}\"\nshards = \"0..64\"\n\ + [[node]]\nname = \"b\"\naddr = \"{b}\"\nshards = \"64..\"\n\ + [[node]]\nname = \"b-spare\"\naddr = \"{spare}\"\nreplica = \"b\"\n" + ); + let _a = start_agreeing(&file, "a", a); + let mut b_node = start_agreeing(&file, "b", b); + let mut spare_node = start_agreeing(&file, "b-spare", spare); + until("an elected leader", || ready(a).contains(r#""leader":""#)); + + ok(a, "POST", "/table/tx", ""); + ok(a, "POST", "/table/tx/field/amount?kind=int&bit_depth=32", ""); + ok(a, "POST", "/table/tx/field/country?kind=set", ""); + ok(a, "POST", "/table/tx/import", "amount 1 5\ncountry 1 GB\n"); + + b_node.close(); + spare_node.close(); + + // A key nobody has seen needs the leader, and `a` can no longer prove it is one. Whether + // `a` led the agreement or followed it makes no difference: a leader's lease is renewed + // by a majority it no longer has, a follower's by a leader it no longer hears. + // + // A fresh key every time. The lease outlives the closes by a moment, and a key interned + // in that moment is a key this node knows from then on - a second attempt with the same + // one would need no leader and prove nothing. + let mut attempt = 100u64; + until("the schema lease to run out", || { + attempt += 1; + let body = format!("amount {attempt} 7\ncountry {attempt} K{attempt}\n"); + let (status, body) = send(a, "POST", "/table/tx/import", &body); + status == 503 && body.contains(r#""code":"schema_lease_lost""#) + }); + + // Still serving its own range: that range has no copy, so nothing could have taken it, + // and the lease it lost is the namespace's and not the range's. + assert!(ready(a).contains(r#""serving":true"#), "{}", ready(a)); + + // A key it already knows needs nobody. The range is its own, and the write lands. (A + // count over the whole table would fan out to `b`'s range, which is the part of the + // cluster that is actually gone - so the import's own answer is the evidence.) + let (status, body) = send(a, "POST", "/table/tx/import", "amount 3 9\ncountry 3 GB\n"); + assert_eq!(status, 200, "{body}"); + assert!(body.contains(r#""imported":2"#), "{body}"); +} + /// **The row-key namespace moves, and nothing is handed the same id twice.** /// /// This is the one change that corrupts rather than fails. A successor that started from what @@ -2059,4 +2441,37 @@ fn the_schema_leader_moves_without_reissuing_anything() { ); let after = ok(b, "POST", "/table/tx/query", "GroupBy(All(), field=\"country\")"); assert!(after.contains("US"), "a key invented after the handover works too: {after}"); + + // **And the old leader refuses to intern**, rather than answering a coordinator that still + // holds the map from before the move. Two nodes interning at once is the one failure that + // hands one string two row ids, and this refusal is what keeps it to one: it used to be + // assumed rather than checked, because the leader was a name in a file. + let fingerprint = + ClusterFile::parse(&file).unwrap().for_node(Some("a"), "").unwrap().fingerprint(); + let stale = |leader: usize| { + big_cluster::wire::InternRequest { + table: "tx".to_string(), + field: "country".to_string(), + keys: vec!["DE".to_string()], + led: Some(big_cluster::wire::Led { epoch: 1, leader }), + } + .encode() + }; + // Sent to `a`, which by its own map no longer leads. + let (status, body) = send_bytes(a, "/internal/intern", &stale(0), fingerprint); + assert_eq!(status, 503, "{}", String::from_utf8_lossy(&body)); + assert!( + String::from_utf8_lossy(&body).contains("not_schema_leader"), + "{}", + String::from_utf8_lossy(&body) + ); + // Sent to `b`, which leads - by a sender that believes `a` does. Refused too: the answer + // names who leads so the sender can ask the right node on purpose rather than by luck. + let (status, body) = send_bytes(b, "/internal/intern", &stale(0), fingerprint); + assert_eq!(status, 503, "{}", String::from_utf8_lossy(&body)); + assert!( + String::from_utf8_lossy(&body).contains("not_schema_leader"), + "{}", + String::from_utf8_lossy(&body) + ); } diff --git a/crates/big-http/tests/operability.rs b/crates/big-http/tests/operability.rs index dcbd3b9..4947719 100644 --- a/crates/big-http/tests/operability.rs +++ b/crates/big-http/tests/operability.rs @@ -163,6 +163,11 @@ fn metrics_render_in_the_prometheus_text_format() { assert!(r.body.contains("big_page_count "), "{}", r.body); // The request that came before this one was counted. assert!(r.body.contains(r#"big_http_responses_total{class="2xx"} 1"#), "{}", r.body); + // The balancer's numbers are there even on a cluster of one, so a dashboard built against + // a single node keeps working when the second node arrives. Off, and nothing moving. + assert!(r.body.contains("big_cluster_balancer_enabled 0"), "{}", r.body); + assert!(r.body.contains("# TYPE big_cluster_ranges_moving gauge"), "{}", r.body); + assert!(r.body.contains("big_cluster_peer_load_unanswered_total 0"), "{}", r.body); // This server is in memory, and `MemPager` counts no I/O because it does none. Absent // rather than zero: a block of zeroes would read as a database nobody is writing to. @@ -208,6 +213,80 @@ fn metrics_report_what_the_storage_backend_did() { assert!(!r.body.contains("big_storage_read_bytes_total"), "{}", r.body); } +/// **A node started with `--reclaim` gives pages back while it is serving**, which nothing in +/// this tree could do before: `big compact` wants the exclusive lock, so shrinking a database +/// meant stopping the daemon. +/// +/// The whole chain, over a real socket: the steward notices the file has enough free space to +/// be worth it, `Api::reclaim` takes the write lock, the pager releases the runs that sit flush +/// against the end of the file, and the counters say what happened. +/// +/// **What this deliberately does not claim.** Only the *tail* is released - one live page near +/// the top pins every free page below it - so a database that is emptied and left alone stays +/// large, and the number here is small. Making the general case shrink means relocating the +/// fragments that sit above the free space, which is not built; the rule that makes the tail +/// drain at all is `the_lowest_reusable_page_is_allocated_first` in `big-pager`. +#[test] +#[cfg(unix)] +fn a_node_with_reclaim_on_gives_pages_back_while_serving() { + let dir = tempfile::tempdir().unwrap(); + let api = Api::open(dir.path().join("t.big")).unwrap(); + api.create_table("tx").unwrap(); + api.create_field("tx", "amount", big_db::catalog::FieldKind::Int, 32).unwrap(); + + // One record per shard rather than a pile in one: a bitmap holds a great many records in a + // handful of pages, and what fills a file is fragments. Enough of them to pass the floor + // the steward will not act below. + let records: Vec = (0..6_000u64).map(|i| i << 20).collect(); + let facts: Vec<_> = records + .iter() + .map(|r| big_embed::Fact::Int { field: "amount", record: *r, value: r % 1_000 }) + .collect(); + api.import("tx", &facts).unwrap(); + assert!(api.metrics().page_count > 4_096, "the fixture has to outgrow the reclaim floor"); + api.delete("tx", &records).unwrap(); + // One more commit. A transaction cannot reuse the pages it is itself freeing - they are + // not past the horizon until it has landed - so the delete's own bookkeeping went to the + // end of the file. The next write takes the lowest free page instead, which is what moves + // the live page off the tail and leaves something to give back. + api.import("tx", &[big_embed::Fact::Int { field: "amount", record: 1, value: 1 }]).unwrap(); + + let server = + Server::bind_with(api, "127.0.0.1:0", ServerConfig { reclaim: true, ..config() }).unwrap(); + let addr = server.local_addr().unwrap(); + let running = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let flag = std::sync::Arc::clone(&running); + let done = std::thread::spawn(move || { + let _ = server.serve_while(&flag); + }); + + // The steward runs on its own clock, so this is a wait rather than an assertion. + let started = std::time::Instant::now(); + loop { + let body = send(addr, "GET", "/metrics", "").body; + assert!(body.contains("# TYPE big_pages_reclaimed_total counter"), "{body}"); + if !body.contains("big_pages_reclaimed_total 0") { + break; + } + assert!( + started.elapsed() < std::time::Duration::from_secs(20), + "nothing was ever given back: {body}" + ); + std::thread::sleep(std::time::Duration::from_millis(50)); + } + + running.store(false, std::sync::atomic::Ordering::Relaxed); + let _ = done.join(); +} + +/// And off by default, which is the half that keeps every existing deployment as it was. +#[test] +fn a_node_without_reclaim_never_gives_a_page_back() { + let addr = spawn(1, config()); + let body = send(addr, "GET", "/metrics", "").body; + assert!(body.contains("big_pages_reclaimed_total 0"), "{body}"); +} + // --------------------------------------------------------------------------- // Credentials // --------------------------------------------------------------------------- diff --git a/docs/clustering.md b/docs/clustering.md index aa64465..f45e557 100644 --- a/docs/clustering.md +++ b/docs/clustering.md @@ -344,6 +344,74 @@ later. This is the same shape as every other refusal in the engine — see the more here than anywhere, because the alternative is two row ids for one string, which is exactly the failure per-node ranges were rejected for. +**A node that has just started has heard from nobody, and says so.** The lease is "when did I +last prove I was in touch with the agreement", and a fresh process has never proved it — so a +node holding a replicated range refuses reads and writes for its range, and a node named as the +schema leader refuses to assign row ids, until the first heartbeat reaches it. On the shipped +clocks that is one election. It is not politeness: a node that restarted may have been replaced +while it was away, and serving on the strength of a clock that also starts at zero is exactly +the two-primaries failure the lease exists to prevent. A cluster with no replicas is unaffected +— nothing there can be taken away, so nothing is fenced. + +**And the leader checks that it still is one.** Being named by the map is not the same as +being allowed to act on it, and a request to intern or allocate is refused, `503`, in three +cases. `not_schema_leader`: the map names somebody else — a coordinator holding a map one +decision old sent this to whoever led *then*, and the answer names who leads now. The request +carries who the sender *thinks* leads, so a sender that is behind is told rather than served. +`not_schema_leader` again: this node was told to stand down by a move that has not landed yet. +And `schema_lease_lost`: this node has lost touch with the agreement and cannot know it has not +been replaced — the same lease a replicated range serves under, and here there is no unfenced +case, because the namespace can always be given to somebody else. What these cost is one thing: +a write with a new key waits. What they prevent is two nodes interning at once, which no write +could wait out. + +**Moving the namespace has a step in the middle where nobody holds it.** `bigctl cluster +schema-leader --to ` copies every row key, then tells the old leader to stop, then reads +its allocation floor — final now, rather than racing — then commits. Between "stop" and the +commit, a write with a new key is refused; that gap is allowed. The gap it replaces, in which +the old leader went on interning for coordinators that had not yet heard, is not. + +**And with `--elect-schema-leader`, the agreement moves it on its own.** Off by default, and +the default is the honest one: a dead schema leader means no *new* row key until it is back, +which is an outage an operator can wait out, and the alternative has a cost that is worth +stating before it is switched on. + +It waits far longer than a range does — fifteen seconds against `promote_after`'s four and a +half — because moving the namespace copies every row key of every table, and a flap would be +the most expensive one available. The successor is chosen the way a range's replacement is: a +voter that is answering and not marked behind, in a fixed order, so two leaders elected in +sequence choose the same node. The decision sets `schema_ready` false, and until the successor +holds every key the survivors have, **nobody interns** — a write with a new key is refused +`schema_handover`, `503`. That gap is the same shape as the manual move's, and it is allowed +for the same reason. + +**What it costs, stated rather than hidden.** The successor's mapping is rebuilt from the +copies every *owner* took when it was told what a key meant — the dead leader is not there to +ask. A key it interned for a write that then failed exists nowhere else, so the successor will +hand that row id to a different string. Inside the surviving cluster that is harmless: nobody +holds the old meaning. It matters only if the deposed node comes back, and there it fails +loudly rather than quietly — `KeyStore::assign` refuses the contradiction, so `POST /repair` +reports it instead of overwriting. That is why the deposed node is marked behind whether or not +it held a copy of anything: **a node the agreement has taken the namespace from must be +repaired before it is trusted again.** Closing this properly means replicating a key assignment +before `intern` returns, which is a replicated catalog and a different design. + +If two *surviving* nodes disagree about what a row id means, the handover stops rather than +picking a side: `schema_ready` stays false, `big_cluster_schema_handover_blocked` goes to 1, +and nothing assigns row ids until somebody looks. A cluster that refuses new keys is +recoverable; a cluster with two meanings for one row id is not. + +**Record ids are reserved through the agreement before they are handed out.** The leader +allocates from a number in its memory, and a successor that started from what is on disk would +re-issue ids the old leader had promised to writes that had not landed — the one failure with no +second chance. So the map carries a ceiling per table, the leader commits a block of 65 536 ids +before it hands out anything under it, and a leader with no floor of its own — just elected, or +just restarted — starts at the ceiling. What a dead leader promised is below it by construction. +What it costs is one entry in the agreement per block, and a gap of unused ids when a leader is +replaced mid-block; record ids are sixty-four bits wide and shard-mapped, so a gap costs nothing. +This is still not consensus over the write path: the ids themselves travel as they always did, +and the agreement records only how far the leader may go. + ## The routes a node uses to reach another Thirteen, all `POST`, all under `/internal/`, all with a binary body and a binary answer. They are diff --git a/runbook.md b/runbook.md index f9ae047..e588f66 100644 --- a/runbook.md +++ b/runbook.md @@ -173,6 +173,14 @@ happen online is the *swap*: `read` hands back a page borrowed straight out of t replacing the file under a live mapping is exactly the dangling reference the pager's four mmap constraints exist to prevent. So the copy runs while serving and only the rename needs the stop. +- **`--reclaim` gives the tail back without stopping anything.** A node started with it hands + trailing free pages to the filesystem while it serves, once a quarter of the file is + reclaimable and it finds a moment with no query in flight. It is not a substitute for + `compact`: only pages *flush against the end* are released, so one live page near the top + pins every free page below it. A file that churns comes down; a file that was emptied and + then left alone stays where it is. `big_pages_reclaimed_total` and + `big_reclaim_blocked_total` are the two numbers - the second climbing while the first stays + flat is a node that is never idle long enough. - **`compact` itself is offline.** It takes the exclusive lock, so stop `big serve` first. Running it against a served database fails with a lock error rather than doing anything behind the daemon's back. - Needs free space for a second copy of the *live data* alongside the original, briefly. @@ -889,6 +897,29 @@ rate(big_storage_writes_total[15m]) / rate(big_txn_id[15m]) > 100 # Configured for full durability but not flushing. The two disagreeing is a bug, not a setting. big_durability{level="none"} == 0 and rate(big_storage_syncs_total[15m]) == 0 and rate(big_txn_id[15m]) > 0 + +# A copy is behind. Redundancy the cluster has lost, and nothing gets it back but POST /repair. +big_cluster_copies_behind > 0 + +# A range has been moving for longer than a move takes. The balancer plans nothing, cluster-wide, +# while anything is marked moving, and nothing clears the mark on its own: `bigctl cluster cancel`. +big_cluster_ranges_moving > 0 for 10m + +# A peer keeps not saying what it weighs. To the balancer it is neither a source nor a +# destination, so a cluster with one silent member quietly stops reshaping. +rate(big_cluster_peer_load_unanswered_total[15m]) > 0 + +# A move landed but its source could not let go of its copy. Space nobody reclaims until somebody +# looks, and the loop that moved it will not say so twice. +increase(big_cluster_balance_drops_failed_total[1h]) > 0 + +# Nobody holds the row-key namespace. A write with a key nobody has seen is refused while this +# is 0, so a handover that does not close in a minute is one that has stalled. +big_cluster_schema_ready == 0 for 1m + +# **Page somebody.** Handing the namespace over found two surviving nodes disagreeing about what +# a row id means. Nothing assigns row ids until this is resolved, and nothing clears it by itself. +big_cluster_schema_handover_blocked > 0 ``` Do **not** alert on `4xx`. It is clients being wrong, which is normal traffic. From 0c036bc3332abba621393802e0f4749efadf9c39 Mon Sep 17 00:00:00 2001 From: 9bany Date: Sat, 5 Sep 2026 10:19:02 +0700 Subject: [PATCH 4/6] fix(pager): give back the pages a transaction allocates and then frees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--reclaim` gave back almost nothing on a churned file: 4594 pages, 4427 of them free, two returned. Cause found by accounting for every page by hand and coming up one short - page 4591, the highest in the file. **A page freed above the tail floor never reached the freelist.** `WriteTxn::free` sends it to a scratch list instead, so `alloc` can hand it straight back to the same transaction, and nothing drained that list at commit: `commit` passes dirty, freelist, roots, snapshots and catalog to `commit_txn` and drops `scratch` on the floor. A transaction whose tree shrinks frees more than it allocates - a leaf merge, a level removed, `free_tree` - so what is left over is counted in `page_count`, materialised in the file, and named by nothing. `truncate_tail` walks down from the end and stops at the first page that is not a reusable free run, so one of those on top pinned all 4425 free pages under it. `Freelist::absorb_scratch` runs at the top of `commit`, before the first allocation `commit_txn` makes: a page flush against the end of the file is given back by lowering `next_pgno` rather than recorded as free, and the rest are pushed with the committing transaction's own id, which keeps them pending for the whole of their own commit and out of reach of the freelist's own allocation loop. Placing it in `commit` rather than `commit_txn` is what makes that ordering true; lowering `next_pgno` after a chain page had been taken from the tail would hand one page number out twice. readme §7.3 carries the argument. **And a counter, so the next one is found rather than deduced.** `Db::audit_pages` accounts for every page in a file as reachable, free, or neither, and `big leaks ` prints it. The scaffold is captured by `Store::begin_audit` under a single `state.read()` - meta, freelist, snapshots and roots are published together by `commit_txn`, and a mark set stitched from two states is wrong in a way no test would catch - and `Db` walks the trees, because `big-pager` cannot see `big-btree`. It holds a `ReadTxn` for the walk and takes no write lock: the counts are exact as of the transaction it opened at, which is all a report needs. The mark covers the class both `scrub` and `copy_to` skip: the trees a snapshot still names through its own old roots chain. `copy_to` skips them because it is building a fresh history; an audit asserts what the existing one needs, so what `copy_to` throws away is what the mark has to add. A bitset rather than a counter, because snapshots share nearly every page with the current trees under copy-on-write and `visit_tree` does not deduplicate. It reports one thing nothing else in the tree checks. A page that is both reachable and in a run older than the horizon has been handed out twice, which is corruption rather than waste - that and a reference past the end of the file are the only two conditions that fail the command. Read-only, and no repair: the mark is taken under a reader, which is sound for reporting and not for acting on. Two things still pin the tail, neither of them a leak, both now written down where the next person will look: the freelist chain a large delete must place at the top of the file, which clears on the next real write; and after that a single live b-tree page, which nothing can move without rewriting whoever points at it. --- crates/big-bin/src/bin/big.rs | 4 + crates/big-bin/src/offline.rs | 68 +++++++- crates/big-db/src/db/mod.rs | 46 ++++++ crates/big-db/tests/leaks.rs | 225 +++++++++++++++++++++++++ crates/big-pager/readme.md | 18 ++ crates/big-pager/src/audit.rs | 227 +++++++++++++++++++++++++ crates/big-pager/src/chainio.rs | 31 ++++ crates/big-pager/src/freelist.rs | 48 ++++++ crates/big-pager/src/lib.rs | 219 ++++++++++++++++++++++++- crates/big-pager/src/txn.rs | 12 +- crates/big-pager/tests/audit.rs | 228 ++++++++++++++++++++++++++ crates/big-pager/tests/reclamation.rs | 52 ++++++ crates/big-pager/tests/store.rs | 55 +++++++ 13 files changed, 1230 insertions(+), 3 deletions(-) create mode 100644 crates/big-db/tests/leaks.rs create mode 100644 crates/big-pager/src/audit.rs create mode 100644 crates/big-pager/tests/audit.rs diff --git a/crates/big-bin/src/bin/big.rs b/crates/big-bin/src/bin/big.rs index 9e3818d..14e50c9 100644 --- a/crates/big-bin/src/bin/big.rs +++ b/crates/big-bin/src/bin/big.rs @@ -51,6 +51,10 @@ Offline, on a file nothing else has open: scrub recompute every checksum can reach opening checks the meta page and the chains; this checks the trees, which nothing on the query path ever does + leaks account for every page: reachable, free, or neither + a page in neither is one nothing will ever read and nothing + will ever reuse; one at the end of the file stops `compact` + giving anything back. reads only, and never repairs A backup is an ordinary database file. Restoring is opening it - `restore` exists so that the procedure has a name, not because the file needs converting. diff --git a/crates/big-bin/src/offline.rs b/crates/big-bin/src/offline.rs index d403dc5..1556ffd 100644 --- a/crates/big-bin/src/offline.rs +++ b/crates/big-bin/src/offline.rs @@ -33,6 +33,7 @@ pub fn main(args: &[String], usage: &str) { ["compact", file] => compact(file), ["verify", file] => verify(file), ["scrub", file] => scrub(file), + ["leaks", file] => leaks(file), ["drop-days", file, table, field, before] => drop_days(file, table, field, before), // Named rather than answered with the usage alone: an operator who typed a real // command with the wrong arguments has made a different mistake from one who typed a @@ -62,7 +63,8 @@ pub fn main(args: &[String], usage: &str) { /// Every command word `big` answers to, for telling a typo from a misuse. `serve` is here /// because it is a real command of this binary, just not one this half handles - so /// `big serve` with no file reaches its own error rather than "no such command". -const KNOWN: [&str; 7] = ["serve", "backup", "restore", "compact", "verify", "drop-days", "scrub"]; +const KNOWN: [&str; 8] = + ["serve", "backup", "restore", "compact", "verify", "drop-days", "scrub", "leaks"]; fn backup(file: &str, dest: &str) -> Result<(), String> { let db = open(file)?; @@ -147,6 +149,70 @@ fn scrub(file: &str) -> Result<(), String> { Ok(()) } +/// Accounts for every page in the file: reachable, free, or neither. +/// +/// `verify` already prints `page_count` and the free counters, but nothing ties them to each +/// other or to the size of the file. This does: it walks every tree the current roots and every +/// snapshot can reach, adds the meta pages, the four chains and the freelist, and reports what +/// is left over. **A page in neither set is one the database will never read and never hand out +/// again**, and one of those at the end of the file is enough to stop `compact` giving anything +/// back to the filesystem. +/// +/// A separate subcommand for the same reason as `scrub`: it costs a full read of the live data, +/// while `verify` costs almost nothing. It only reads - there is no repair here, and the counts +/// it reports are exact as of the transaction it opened at rather than of any later one. +/// +/// This is not a scrub. The walk parses pages without verifying their checksums, so run +/// `big scrub ` first if the file is under suspicion. +fn leaks(file: &str) -> Result<(), String> { + let db = open(file)?; + let r = db.audit_pages().map_err(|e| format!("{file} could not be audited: {e}"))?; + + println!("{file}: txn {}, {} pages", r.txn_id, r.page_count); + println!( + " {} reachable ({} meta, {} chain, {} tree, {} snapshot chain, {} snapshot tree)", + r.reachable, + r.by_class.meta, + r.by_class.chains, + r.by_class.trees, + r.by_class.snapshot_chains, + r.by_class.snapshot_trees + ); + println!( + " {} free ({} reusable now, {} still pending)", + r.free_total, + r.free_reusable, + r.free_total - r.free_reusable + ); + println!(" {} unaccounted for", r.leaked); + + if r.leaked > 0 { + println!( + " highest is {}, of {} pages: {:?}", + r.highest_leaked.map_or("none".to_string(), |p| p.to_string()), + r.page_count, + r.leaked_sample + ); + println!(" `big compact {file}` rewrites the file without them"); + } + if r.file_pages != r.page_count { + println!( + " {} pages past what the meta page records ({} on disk against {})", + r.beyond_meta, r.file_pages, r.page_count + ); + println!(" a crash between a meta flip and a truncation leaves these; harmless, and `compact` clears them"); + } + + // Only these two are corruption rather than waste, and only these two fail the command. + if r.double_allocated > 0 || r.dangling > 0 { + return Err(format!( + "{file} is corrupt: {} page(s) handed out twice ({:?}), {} reference(s) past the end of the file", + r.double_allocated, r.double_allocated_sample, r.dangling + )); + } + Ok(()) +} + /// Opening a path that does not exist would *create* an empty database - the pager opens with /// `create(true)` because that is what a server wants on first start. A tool that inspects or /// copies must not, so the check is here rather than there. diff --git a/crates/big-db/src/db/mod.rs b/crates/big-db/src/db/mod.rs index a948883..e8e2d5b 100644 --- a/crates/big-db/src/db/mod.rs +++ b/crates/big-db/src/db/mod.rs @@ -128,6 +128,52 @@ impl Db

{ Ok(total) } + /// Counts the pages this file contains that nothing points at and nothing has recorded as + /// free. + /// + /// **It changes nothing.** A page like that is space the file will never use and never + /// give back, and - because [`Db::reclaim`] can only release pages flush against the end - + /// one of them at the top pins every free page beneath it. This says how many there are; + /// it does not free them, and the page numbers it reports must not be fed to anything that + /// does. See [`big_pager::LeakReport::leaked_sample`]. + /// + /// **Cost is a full read of the live data**, the same as [`Db::scrub`] or a backup, and it + /// holds a read transaction throughout - so the reclaim horizon does not move while it + /// runs. Not for a request path and not for a timer. + /// + /// This is not a scrub: the walk parses every page it reads but does not verify checksums, + /// except on the chains, where an unverified `next` would send the walk to an arbitrary + /// page and count it as live. Run `big scrub` for the checksums. + /// + /// Any error aborts the whole audit rather than skipping a tree. A mark phase that + /// swallowed an error would under-mark, and under-marking reports leaks that are not + /// there - a wrong number presented as a fact is worse than a failure. + pub fn audit_pages(&self) -> Result { + let mut audit = self.store.begin_audit()?; + + // The current trees. `visit_tree` yields branch pages, leaf pages and every page a + // leaf cell owns - dense bitmaps, the base page of a delta, and a columnar segment's + // spill page. Columnar segments share the root-record namespace, so they are already + // in `roots` and need no separate enumeration. + for root in core::mem::take(&mut audit.roots) { + big_btree::visit_tree(self.store.pager(), root, &mut |pgno, _| { + audit.mark_tree_page(pgno, false); + Ok(()) + })?; + } + + // **And the trees only a snapshot still names.** These are unreachable from the + // current roots and entirely live; `scrub` and `copy_to` both walk past them. + for root in core::mem::take(&mut audit.snapshot_roots) { + big_btree::visit_tree(self.store.pager(), root, &mut |pgno, _| { + audit.mark_tree_page(pgno, true); + Ok(()) + })?; + } + + Ok(audit.finish()) + } + /// What the row-key dictionary costs this process right now. /// /// Every row key is resident, in both directions, for the life of the process: a key has diff --git a/crates/big-db/tests/leaks.rs b/crates/big-db/tests/leaks.rs new file mode 100644 index 0000000..2e6e3c5 --- /dev/null +++ b/crates/big-db/tests/leaks.rs @@ -0,0 +1,225 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Every page of a real database accounted for. +//! +//! `crates/big-pager/tests/audit.rs` proves the counting; this proves the *walk* - that the +//! mark phase reaches every class of page a `Db` actually produces. A class it misses is a +//! class it reports as leaked, so each test below is one page class that would otherwise be +//! silently wrong. + +use big_db::*; +use big_pager::MemPager; + +fn audit(d: &Db) -> big_pager::LeakReport { + d.audit_pages().expect("the file is readable") +} + +/// Fragments spread one per shard: what fills a file is fragments, not records. +fn seeded(shards: u64) -> Db { + let d = Db::in_memory().unwrap(); + d.create_table("tx").unwrap(); + d.create_field("tx", "amount", FieldKind::Int, 32).unwrap(); + d.create_field("tx", "country", FieldKind::Set, 0).unwrap(); + let mut w = d.write(); + for i in 0..shards { + let record = i << 20; + w.set_int("tx", "amount", record, i * 7).unwrap(); + w.set_key("tx", "country", record, if i.is_multiple_of(3) { "vn" } else { "jp" }).unwrap(); + } + w.commit().unwrap(); + d +} + +/// **The number that must stay zero.** Everything below is one way of making it non-zero. +#[test] +fn a_fresh_database_accounts_for_every_page() { + let d = Db::in_memory().unwrap(); + let r = audit(&d); + assert!(r.is_clean(), "{r:?}"); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +#[test] +fn a_database_with_data_accounts_for_every_page() { + let d = seeded(200); + let r = audit(&d); + assert_eq!(r.leaked, 0, "{r:?}"); + assert_eq!(r.dangling, 0, "{r:?}"); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +/// **A columnar table's spill pages.** A leaf cell of kind `ValuesPtr` owns a page the way a +/// dense bitmap cell does; a walk that only followed bitmaps would report every one of them as +/// leaked, which is the loudest possible failure and therefore the best test. +#[test] +fn a_columnar_table_leaks_no_spill_pages() { + let d = Db::in_memory().unwrap(); + d.create_table_with("tx", TableEngine::Columnar).unwrap(); + d.create_field("tx", "amount", FieldKind::Int, 32).unwrap(); + let mut w = d.write(); + // Enough distinct values in one shard that the column spills to a page of its own. + for r in 0..5_000u64 { + w.set_int("tx", "amount", r, r * 31).unwrap(); + } + w.commit().unwrap(); + + let r = audit(&d); + assert_eq!(r.leaked, 0, "a columnar spill page was counted as leaked: {r:?}"); + assert!(r.by_class.trees > 0, "{r:?}"); +} + +/// **A delta cell's base page.** Writing a fragment twice leaves a cell that names the page +/// holding what it is a delta against. `visit_tree` follows it; a hand-rolled walk forgets it. +#[test] +fn a_fragment_written_twice_leaks_no_delta_base() { + let d = seeded(64); + let mut w = d.write(); + for i in 0..64u64 { + w.set_int("tx", "amount", (i << 20) + 1, i * 13).unwrap(); + } + w.commit().unwrap(); + + let r = audit(&d); + assert_eq!(r.leaked, 0, "{r:?}"); +} + +/// A database that has been mostly emptied still accounts for every page: what the delete +/// freed is in the freelist, and what is left is reachable. +#[test] +fn a_database_that_has_churned_leaks_nothing() { + let d = seeded(900); + let doomed: Vec = (0..900u64).filter(|i| !i.is_multiple_of(50)).map(|i| i << 20).collect(); + let mut w = d.write(); + w.delete("tx", &doomed).unwrap(); + w.commit().unwrap(); + let mut w = d.write(); + w.set_int("tx", "amount", 1, 1).unwrap(); + w.commit().unwrap(); + + let r = audit(&d); + assert_eq!( + r.leaked, 0, + "pages nothing points at and nothing recorded as free; highest is {:?}: {r:?}", + r.highest_leaked + ); +} + +/// **What `truncate_tail` stops at is always a page somebody owns.** +/// +/// It walks down from the end of the file and halts at the first page that is not a reusable +/// free run, so whatever sits on top pins every free page beneath it - on this fixture, thousands +/// of them. Before `Freelist::absorb_scratch` the page on top was routinely one that *nobody* +/// owned, and there was no way to give it back short of rewriting the whole file. +/// +/// This asserts the part that is now structural: reclaim may still stall, but never on a page the +/// file has lost track of. It deliberately does not assert how much comes back, because that is +/// not yet a property of the engine - see below. +/// +/// **What still pins the tail, measured on this fixture and in this order.** Neither is a leak; +/// the audit is right to report zero for both. +/// +/// 1. *The freelist chain of a large delete.* A transaction cannot reuse the pages it is itself +/// freeing - they carry its own id, which is above the horizon until it has committed - so the +/// commit that frees thousands of pages must put its own freelist chain at the top of the file. +/// An unchanged chain is then kept where it lies, so empty commits never move it; one real +/// write does, and the tail drops by the size of the chain. +/// 2. *A single live b-tree page.* Whatever is highest after that pins everything below it for +/// good. Moving it means rewriting whoever points at it, which is knowledge `big-btree` has and +/// `big-pager` does not - so `truncate_tail` cannot, by construction, and this is the work that +/// would make reclaim give back a churned file. +#[test] +fn reclaim_never_stalls_on_a_page_nobody_owns() { + let d = seeded(900); + let doomed: Vec = (0..900u64).filter(|i| !i.is_multiple_of(50)).map(|i| i << 20).collect(); + let mut w = d.write(); + w.delete("tx", &doomed).unwrap(); + w.commit().unwrap(); + let mut w = d.write(); + w.set_int("tx", "amount", 1, 1).unwrap(); + w.commit().unwrap(); + + d.store().truncate_tail().expect("no readers are alive here"); + + let r = audit(&d); + assert_eq!(r.leaked, 0, "reclaim stopped at a page nothing owns: {r:?}"); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +/// **The check nothing else in the tree makes.** A page both reachable and reusable-free has +/// been handed out twice. Reachable-and-*pending*-free is ordinary - that is what a snapshot +/// pinning an old tree looks like - so only the reusable half is evidence. +#[test] +fn no_page_is_both_live_and_free_to_reuse() { + let d = seeded(300); + let mut w = d.write(); + w.delete("tx", &(0..150u64).map(|i| i << 20).collect::>()).unwrap(); + w.commit().unwrap(); + + let r = audit(&d); + assert_eq!(r.double_allocated, 0, "a page was handed out twice: {r:?}"); + assert!(r.double_allocated_sample.is_empty(), "{r:?}"); +} + +/// The audit is exact as of the transaction it opened at, and stays right while writers commit +/// underneath it. This is the test that catches a scaffold captured across two different +/// states, and it is the only one that would. +#[test] +fn the_audit_is_stable_while_a_writer_commits() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + let d = Arc::new(seeded(120)); + let stop = Arc::new(AtomicBool::new(false)); + + let writer = { + let d = Arc::clone(&d); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + let mut n = 0u64; + while !stop.load(Ordering::Relaxed) { + let mut w = d.write(); + w.set_int("tx", "amount", (n % 120) << 20, n).unwrap(); + w.commit().unwrap(); + n += 1; + } + n + }) + }; + + for _ in 0..12 { + let r = audit(&d); + assert_eq!(r.leaked, 0, "{r:?}"); + assert_eq!(r.dangling, 0, "{r:?}"); + assert_eq!(r.double_allocated, 0, "{r:?}"); + } + stop.store(true, Ordering::Relaxed); + let commits = writer.join().unwrap(); + assert!(commits > 0, "the writer never got a turn, so this proved nothing"); +} + +/// A compact copy is a fresh history: it must account for every page it has. +#[test] +fn a_compacted_copy_accounts_for_every_page() { + let src = seeded(200); + let mut w = src.write(); + w.delete("tx", &(0..100u64).map(|i| i << 20).collect::>()).unwrap(); + w.commit().unwrap(); + + let copy = src.copy_to(MemPager::new()).unwrap(); + let r = audit(©); + assert_eq!(r.leaked, 0, "{r:?}"); + assert_eq!(r.free_total, 0, "a fresh copy has nothing free: {r:?}"); + assert_eq!(r.reachable, r.page_count, "{r:?}"); +} diff --git a/crates/big-pager/readme.md b/crates/big-pager/readme.md index 70dbd37..0721008 100644 --- a/crates/big-pager/readme.md +++ b/crates/big-pager/readme.md @@ -445,6 +445,24 @@ in one or two passes. *Test:* `repeated_commits_do_not_grow_the_file_without_bound`. +**Pages a transaction gives back to itself feed this loop, and safely.** `WriteTxn::free` sends a +page the transaction itself allocated to a scratch list rather than the freelist, so `alloc` can +hand it straight back; `commit` calls `Freelist::absorb_scratch` on what is left, which lowers +`next_pgno` past pages flush with the end of the file and pushes the rest. **Without that, those +pages were lost outright:** materialised in the file, counted in `meta.page_count`, and named by +nothing — and a single one of them at the end of the file stops §7.4 releasing anything at all. +Every transaction whose tree shrinks produces some. + +They arrive stamped `freed_at = txn_id`, the committing transaction's own, which is above the +horizon by construction (all three horizon terms are `.min(base.txn_id)`). So they inflate +`entry_count` **without being allocatable**, and a shrinking commit may take one more chain page +from the tail than it otherwise would; the next commit, whose horizon has moved, takes it back. +The argument above is untouched — it rests on `alloc` taking from the front of a run, and +`absorb_scratch` only ever calls `push`. + +*Tests:* `a_transaction_that_abandons_its_own_pages_leaves_no_trace`, +`repeated_churning_commits_do_not_grow_the_file_without_bound`. + ### 7.4 Returning space to the filesystem `truncate_tail` drops reclaimable runs that sit flush against the end of the file, repeatedly, diff --git a/crates/big-pager/src/audit.rs b/crates/big-pager/src/audit.rs new file mode 100644 index 0000000..3556128 --- /dev/null +++ b/crates/big-pager/src/audit.rs @@ -0,0 +1,227 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Accounting for every page in the file: reachable, free, or neither. +//! +//! **Nothing here frees anything.** It answers one question - *how many pages does this file +//! contain that nothing points at and nothing has recorded as free* - and answers it as a +//! number, not as an action. A page like that is invisible to every other measurement: it is +//! counted in `page_count`, so the file is that much bigger, and it is in no freelist run, so +//! it is never handed out again. It is also enough, on its own, to stop +//! [`crate::Store::truncate_tail`] releasing anything at all - that walks down from the end of +//! the file and stops at the first page that is not a reusable free run, so one orphan at the +//! top pins every free page beneath it. +//! +//! **The complement is only as good as the mark.** A page class the mark phase forgets is a +//! page class this reports as leaked, so the enumeration in [`crate::Store::begin_audit`] is +//! the whole correctness argument, and the class that is easiest to forget is the one no other +//! walk in this tree visits: the trees an old *snapshot* still names. `Db::copy_to` skips them +//! deliberately, because it is building a new history; an audit is asserting what the existing +//! history still needs, so the one thing that walk drops is the one thing this must add. + +use crate::freelist::FreeRun; +use big_page::{Pgno, TxnId}; + +/// One bit per page. +/// +/// **A bitset rather than a count, and that is not an optimisation.** Under copy-on-write a +/// snapshot's tree shares almost every page with the current one, so the same page is +/// legitimately reached many times over across the root set - a counter would multiply +/// reachability and report a negative number of leaks. [`big_btree::visit_tree`] does not +/// deduplicate either: a page reached through two cells is yielded twice. Marking an +/// already-marked bit is a no-op, so neither matters here. +/// +/// [`big_btree::visit_tree`]: https://docs.rs/big-btree +#[derive(Clone, Debug)] +pub struct PageSet { + words: Vec, + len: u64, +} + +impl PageSet { + /// A set that can hold every page below `pages`, all of them clear. + pub fn with_pages(pages: u64) -> Self { + Self { words: vec![0; pages.div_ceil(64) as usize], len: pages } + } + + /// How many pages this set is able to describe. Anything at or above it is out of the + /// question being asked, not a member that happens to be absent. + pub fn capacity(&self) -> u64 { + self.len + } + + /// Marks a page. `false` when the page is outside this set, which a caller counts rather + /// than ignores: a reference past the end of the file is a fact worth reporting. + pub fn insert(&mut self, pgno: Pgno) -> bool { + let at = pgno as u64; + if at >= self.len { + return false; + } + self.words[(at / 64) as usize] |= 1 << (at % 64); + true + } + + pub fn contains(&self, pgno: Pgno) -> bool { + let at = pgno as u64; + at < self.len && self.words[(at / 64) as usize] & (1 << (at % 64)) != 0 + } + + /// How many pages are marked. + pub fn count(&self) -> u64 { + self.masked_words().map(|w| w.count_ones() as u64).sum() + } + + /// Pages in neither this set nor `other`. + pub fn count_absent_from_both(&self, other: &PageSet) -> u64 { + // `& mask` on the complement, not on the inputs. Inverting a word turns every bit past + // the end of the file into a page that is absent from both sets, and a file whose page + // count is not a multiple of 64 would report up to 63 leaks that are not pages. + self.pair(other).map(|(a, b, mask)| ((!(a | b)) & mask).count_ones() as u64).sum() + } + + /// Every page in neither set, lowest first. `DoubleEndedIterator`, so the *highest* - the + /// one that pins the tail - is one `next_back` away. + pub fn absent_from_both<'a>( + &'a self, + other: &'a PageSet, + ) -> impl DoubleEndedIterator + 'a { + let pages: Vec = self + .pair(other) + .enumerate() + .flat_map(|(i, (a, b, mask))| { + let free = (!(a | b)) & mask; + (0..64u64).filter_map(move |bit| { + (free & (1 << bit) != 0).then_some((i as u64 * 64 + bit) as Pgno) + }) + }) + .collect(); + pages.into_iter() + } + + /// Pages in both sets, lowest first. + pub fn present_in_both<'a>(&'a self, other: &'a PageSet) -> impl Iterator + 'a { + self.pair(other).enumerate().flat_map(|(i, (a, b, mask))| { + let both = a & b & mask; + (0..64u64).filter_map(move |bit| { + (both & (1 << bit) != 0).then_some((i as u64 * 64 + bit) as Pgno) + }) + }) + } + + /// The words of both sets, each with the mask of the pages that actually exist. + /// + /// The mask is handed out rather than pre-applied because the caller that needs it most is + /// the one taking a *complement*: masking the inputs does nothing for `!(a | b)`, which + /// sets every bit past the end of the file. A file whose page count is not a multiple of + /// 64 would then report up to 63 leaks that are not pages at all. + fn pair<'a>(&'a self, other: &'a PageSet) -> impl Iterator + 'a { + debug_assert_eq!(self.len, other.len, "two page sets over different files"); + let masks = self.masks(); + self.words.iter().zip(other.words.iter()).zip(masks).map(|((a, b), m)| (*a, *b, m)) + } + + /// One mask per word: all ones, except the last, which keeps only the pages that exist. + fn masks(&self) -> impl Iterator + '_ { + let last = (self.len % 64) as u32; + let tail = self.words.len().saturating_sub(1); + (0..self.words.len()).map(move |i| { + if i == tail && last != 0 { + (1u64 << last) - 1 + } else { + u64::MAX + } + }) + } + + fn masked_words(&self) -> impl Iterator + '_ { + self.words.iter().zip(self.masks()).map(|(w, m)| w & m) + } +} + +/// How many pages of each kind the mark phase reached. Every figure counts *marks made*, so a +/// page shared between a snapshot and the current tree is counted under both. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub struct AuditTally { + pub meta: u64, + pub chains: u64, + pub snapshot_chains: u64, + pub trees: u64, + pub snapshot_trees: u64, +} + +/// What an audit found. Every number is exact as of [`LeakReport::txn_id`] and no later: the +/// audit holds one read transaction from start to finish, so a commit that lands while it runs +/// is simply not part of the question it answered. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct LeakReport { + /// The transaction the file was at when the audit began. + pub txn_id: TxnId, + /// `meta.page_count` - the universe of the question. + pub page_count: u64, + /// What the backing file actually holds. Larger than `page_count` means pages the meta + /// page does not account for; see [`LeakReport::beyond_meta`]. + pub file_pages: u64, + /// Pages something points at. + pub reachable: u64, + /// Pages the freelist holds, reusable or not. + pub free_total: u64, + /// The subset of those a writer could take right now. + pub free_reusable: u64, + /// **Pages nothing points at and nothing has recorded as free.** Space the file will never + /// use and never give back. + pub leaked: u64, + /// The highest of them. This is the number that explains a `truncate_tail` that released + /// nothing: every free page below it is pinned by it. + pub highest_leaked: Option, + /// The lowest few, for somebody who wants to go and look at them. + /// + /// **Not an input to anything that frees.** These page numbers were derived under a read + /// transaction while writers were free to commit; a later commit may have put any of them + /// to work. Acting on this list requires re-deriving it under the write lock with no + /// readers alive, the way `Store::truncate_tail` does. + pub leaked_sample: Vec, + /// Pages in the file past what the meta page accounts for. Nonzero after a crash between + /// the meta write and the truncation in `truncate_tail`; those pages are absorbed on the + /// next open and never recovered. + pub beyond_meta: u64, + /// References to pages at or beyond the end of the file. **Always a damaged file.** + pub dangling: u64, + /// Pages both reachable and *reusable*-free - a page that has been handed out twice. + /// + /// Reachable-and-free on its own is ordinary: a page a snapshot pins is reachable from + /// that snapshot's tree and sits in the freelist as a pending run. Restricting to reusable + /// runs removes exactly that case, because a snapshot at transaction `S` holds the horizon + /// at or below `S` while everything its tree reaches was freed after `S`. What is left is + /// corruption, and nothing else in this tree looks for it. + pub double_allocated: u64, + pub double_allocated_sample: Vec, + pub by_class: AuditTally, +} + +impl LeakReport { + /// Whether the file accounts for every page it contains. + pub fn is_clean(&self) -> bool { + self.leaked == 0 + && self.beyond_meta == 0 + && self.dangling == 0 + && self.double_allocated == 0 + } +} + +/// How many pages a run covers, clamped into the file. +pub(crate) fn run_pages(run: &FreeRun, limit: u64) -> impl Iterator { + let first = run.first as u64; + let end = (first + run.len as u64).min(limit); + (first..end).map(|p| p as Pgno) +} diff --git a/crates/big-pager/src/chainio.rs b/crates/big-pager/src/chainio.rs index f4735c5..80ef791 100644 --- a/crates/big-pager/src/chainio.rs +++ b/crates/big-pager/src/chainio.rs @@ -43,6 +43,37 @@ pub fn load_chain( Ok(out) } +/// The same walk as [`chain_pgnos`], with every page's checksum verified before its `next` is +/// followed. +/// +/// **For a caller that is auditing rather than recycling.** `chain_pgnos` trusts `next`, which +/// is right on the write path: it is reading a chain this process just wrote, and a checksum +/// failure there would turn a damaged file into a refused *commit*. An audit is the opposite +/// case - it reads a file it has no reason to trust, and an unverified `next` sends the walk to +/// an arbitrary page which it would then count as live. That under-reports, which is the +/// silent direction, so this one pays for the checksum. +pub fn chain_pgnos_checked( + pager: &P, + root: Option, + ty: PageType, + stride: usize, +) -> Result> { + let mut out = Vec::new(); + let mut cur = root; + let limit = pager.page_count() + 1; + + while let Some(pgno) = cur { + if out.len() as u64 > limit { + return Err(StoreError::ChainCycle { root: root.unwrap_or(0) }); + } + out.push(pgno); + let page = pager.read(pgno)?; + page.verify_checksum()?; + cur = ChainPage::parse(&page, ty, stride)?.next(); + } + Ok(out) +} + /// Walks the chain and returns every page number, needed to hand them back to the freelist /// when the chain is rewritten. pub fn chain_pgnos( diff --git a/crates/big-pager/src/freelist.rs b/crates/big-pager/src/freelist.rs index 924cca5..7b07fbe 100644 --- a/crates/big-pager/src/freelist.rs +++ b/crates/big-pager/src/freelist.rs @@ -85,6 +85,54 @@ impl Freelist { self.runs.push(FreeRun { freed_at, first: pgno, len: 1 }); } + /// Takes back the pages a transaction both allocated and freed, and answers with the + /// `next_pgno` it should commit. + /// + /// **Without this they are lost.** A page a transaction allocates from the tail and then + /// frees never reaches the freelist - it goes to the transaction's scratch list, to be + /// handed straight back out if the same transaction allocates again. A transaction whose + /// tree *shrinks* allocates fewer times than it frees, so what is left over at commit is + /// counted in `page_count`, materialised in the file, and referenced by nothing: a page + /// the database will never read and never reuse. One of those at the end of the file is + /// enough to stop [`crate::Store::truncate_tail`] releasing anything at all. + /// + /// A page flush against the end is given back by lowering `next_pgno` rather than by being + /// recorded as free - the file simply never grows that far. Everything else is pushed. + /// + /// `freed_at` must be the committing transaction's own id, never lower: all three horizon + /// computations are at or below the *base* transaction, so a page stamped with this one is + /// pending for the whole of its own commit and cannot be handed out by the allocations + /// that follow - including the freelist's own, whose termination argument (readme §7.3) + /// assumes the entry count only falls while it runs. + pub fn absorb_scratch( + &mut self, + mut scratch: Vec, + next_pgno: u64, + freed_at: TxnId, + ) -> u64 { + if scratch.is_empty() { + return next_pgno; + } + scratch.sort_unstable(); + // Freeing one page twice inside a transaction is a bug above this line either way, and + // it is already one today: `alloc` would hand the same page out twice. Collapsing it + // here makes the consequence the survivable one - a leak rather than two owners of one + // page - while the assertion still fails the test that produced it. + let before = scratch.len(); + scratch.dedup(); + debug_assert_eq!(before, scratch.len(), "a page was freed twice inside one transaction"); + + let mut next = next_pgno; + while scratch.last().is_some_and(|p| *p as u64 + 1 == next) { + scratch.pop(); + next -= 1; + } + for p in scratch { + self.push(p, freed_at); + } + next + } + /// Merges adjacent runs sharing a `freed_at`. This is where the run-length encoding pays off. /// /// **Ordered by page number, and that is what makes the file able to shrink.** It used to diff --git a/crates/big-pager/src/lib.rs b/crates/big-pager/src/lib.rs index 4cc4f30..43baadf 100644 --- a/crates/big-pager/src/lib.rs +++ b/crates/big-pager/src/lib.rs @@ -27,6 +27,7 @@ #![deny(unsafe_code)] +pub mod audit; pub mod chainio; #[cfg(feature = "counting-pager")] pub mod counting; @@ -44,6 +45,7 @@ pub mod txn; #[cfg(unix)] pub mod mmap; +pub use audit::{AuditTally, LeakReport, PageSet}; #[cfg(feature = "counting-pager")] pub use counting::{CountingPager, PagerCounts}; pub use durability::Durability; @@ -73,7 +75,7 @@ pub use big_page::{ use big_page::{ build_chain, chain_pages_needed as pages_needed, pick_meta, PageType, ROOT_RECORD_BYTES, }; -use chainio::{chain_pgnos, load_chain}; +use chainio::{chain_pgnos, chain_pgnos_checked, load_chain}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex, MutexGuard, RwLock}; @@ -191,6 +193,82 @@ struct StoreState { freelist: Freelist, } +/// An audit in progress: the file's own bookkeeping, captured at one instant, plus the reader +/// that keeps it true while the caller walks the trees. +/// +/// Everything reachable without leaving this crate is already marked. The caller marks the +/// trees - `roots` and then `snapshot_roots`, both of which need `big-btree` - and calls +/// [`Audit::finish`]. +pub struct Audit<'db, P: Pager> { + /// Held for the life of the audit. Dropping it early would let the horizon advance and a + /// page named in the capture be recycled while the walk is still reading it. + _read: ReadTxn<'db, P>, + /// Pages something points at. The caller adds the trees. + pub marks: PageSet, + /// Pages the freelist holds, pending runs included. + pub free: PageSet, + /// Roots of the current trees, for the caller to walk. + pub roots: Vec, + /// Roots reachable only from a snapshot's old roots chain. **The class every other walk in + /// this tree skips**, and the one an audit must not. + pub snapshot_roots: Vec, + /// Runs a writer could take right now. Anything both reachable and in one of these has + /// been handed out twice. + reusable: Vec, + tally: AuditTally, + dangling: u64, + free_total: u64, + free_reusable: u64, + page_count: u64, + file_pages: u64, + txn_id: TxnId, +} + +impl Audit<'_, P> { + /// Marks a page the caller reached, counting it against a tree or a snapshot's tree. + pub fn mark_tree_page(&mut self, pgno: Pgno, from_snapshot: bool) { + if self.marks.insert(pgno) { + if from_snapshot { + self.tally.snapshot_trees += 1; + } else { + self.tally.trees += 1; + } + } else if pgno as u64 >= self.page_count { + self.dangling += 1; + } + } + + /// The complement, once the caller has marked everything it can reach. + pub fn finish(self) -> LeakReport { + const SAMPLE: usize = 64; + let leaked: Vec = self.marks.absent_from_both(&self.free).take(SAMPLE).collect(); + let mut reusable = PageSet::with_pages(self.page_count); + for run in &self.reusable { + for p in audit::run_pages(run, self.page_count) { + reusable.insert(p); + } + } + let doubled: Vec = self.marks.present_in_both(&reusable).take(SAMPLE).collect(); + + LeakReport { + txn_id: self.txn_id, + page_count: self.page_count, + file_pages: self.file_pages, + reachable: self.marks.count(), + free_total: self.free_total, + free_reusable: self.free_reusable, + leaked: self.marks.count_absent_from_both(&self.free), + highest_leaked: self.marks.absent_from_both(&self.free).next_back(), + leaked_sample: leaked, + beyond_meta: self.file_pages.saturating_sub(self.page_count), + dangling: self.dangling, + double_allocated: self.marks.present_in_both(&reusable).count() as u64, + double_allocated_sample: doubled, + by_class: self.tally, + } + } +} + /// Bound only by `Pager`/`PagerMut`, so swapping the backend touches nothing in here. pub struct Store { pager: P, @@ -326,6 +404,137 @@ impl Store

{ Ok(ReadTxn::new(self, snap.txn_id, Arc::new(roots), catalog)) } + /// Opens an audit: a reader, plus everything about the file's own bookkeeping taken at the + /// same instant. + /// + /// **The capture has to happen here, under one lock.** A caller assembling the same facts + /// from `meta()`, `snapshots()` and the freelist would take three separate read guards, and + /// `commit_txn` publishes all of them under one write guard - so a mark set built from a + /// straddled state would be wrong in a way no test reliably catches. + /// + /// This marks everything it can reach without leaving this crate: the meta pages, the four + /// chains, and every chain a snapshot still names. What it cannot do is walk a b-tree - + /// `big-btree` depends on this crate, not the other way round - so it hands back the roots + /// and leaves that to the caller. See `Db::audit_pages`. + /// + /// The read transaction is held for the whole audit, which pins the reclaim horizon just as + /// a backup does: `truncate_tail` will refuse for the duration, and any page named in this + /// capture is safe from being recycled underneath the walk. + pub fn begin_audit(&self) -> Result> { + // One guard. The reader is registered before it is dropped, so nothing committed + // between the capture and the registration can move the horizon past us. + let (read, meta, freelist, snapshots, roots, horizon) = { + let st = self.state.read().unwrap(); + let reader_h = self.oldest_reader().unwrap_or(st.meta.txn_id); + let snap_h = st.snapshots.oldest_txn_id().unwrap_or(st.meta.txn_id); + let read = + ReadTxn::new(self, st.meta.txn_id, Arc::clone(&st.roots), Arc::clone(&st.catalog)); + ( + read, + st.meta, + st.freelist.clone(), + Arc::clone(&st.snapshots), + Arc::clone(&st.roots), + reader_h.min(snap_h).min(st.meta.txn_id), + ) + }; + + let pages = meta.page_count; + let mut marks = PageSet::with_pages(pages); + let mut free = PageSet::with_pages(pages); + let mut tally = AuditTally::default(); + let mut dangling = 0u64; + + // 1. Both meta slots. They are double-buffered - a commit writes only `txn_id % 2` - + // so the one this file is not currently governed by still holds the previous commit + // and is what `Store::load` falls back to. + for slot in 0..META_PAGES { + if marks.insert(slot as Pgno) { + tally.meta += 1; + } + } + + // 2. The four chains' own pages. + for (head, ty, stride) in Self::chain_shapes(&meta) { + for p in chain_pgnos_checked(&self.pager, head, ty, stride)? { + if marks.insert(p) { + tally.chains += 1; + } else { + dangling += 1; + } + } + } + + // 3. Every snapshot's *old* roots chain, and the roots on it. This is the class no + // other walk in the tree visits: those trees are unreachable from the current roots + // and entirely live. `0` is the absent marker, the same one `begin_read_at` uses. + let mut snapshot_roots = Vec::new(); + for snap in snapshots.all() { + let head = (snap.root_records != 0).then_some(snap.root_records); + if head.is_none() { + continue; + } + for p in + chain_pgnos_checked(&self.pager, head, PageType::RootRecords, ROOT_RECORD_BYTES)? + { + if marks.insert(p) { + tally.snapshot_chains += 1; + } else { + dangling += 1; + } + } + let old = RootRecords::from_entries(&load_chain( + &self.pager, + head, + PageType::RootRecords, + ROOT_RECORD_BYTES, + )?); + snapshot_roots.extend(old.iter().map(|(_, p)| *p)); + } + + // 4. Everything the freelist holds, pending runs included: pending is not reusable, + // but it is certainly not unaccounted for. + let mut free_total = 0u64; + let mut reusable = Vec::new(); + for run in freelist.runs() { + for p in audit::run_pages(run, pages) { + if free.insert(p) { + free_total += 1; + } + } + if run.freed_at <= horizon { + reusable.push(*run); + } + } + + Ok(Audit { + _read: read, + marks, + free, + roots: roots.iter().map(|(_, p)| *p).collect(), + snapshot_roots, + reusable, + tally, + dangling, + free_total, + free_reusable: freelist.reusable_pages(horizon), + page_count: pages, + file_pages: self.pager.page_count(), + txn_id: meta.txn_id, + }) + } + + /// The four chains, as `(head, type, stride)`. One list, so a caller cannot walk three of + /// them and forget the fourth. + fn chain_shapes(meta: &MetaPage) -> [(Option, PageType, usize); 4] { + [ + (meta.root_records, PageType::RootRecords, ROOT_RECORD_BYTES), + (meta.freelist, PageType::Freelist, FREE_ENTRY_BYTES), + (meta.snapshots, PageType::Snapshots, SNAPSHOT_ENTRY_BYTES), + (meta.catalog, PageType::Catalog, CATALOG_ENTRY_BYTES), + ] + } + pub(crate) fn register_reader(&self, txn_id: TxnId) { *self.readers.lock().unwrap().entry(txn_id).or_insert(0) += 1; } @@ -622,6 +831,14 @@ impl Store

{ txn_id, )?; let free_pgnos = alloc_freelist_pages(&mut freelist, horizon, next_pgno, cap)?; + // The freelist is about to be written down as the file's record of what is spare, so + // nothing in it may name a page beyond where the file ends. Guards the old invariant as + // much as `absorb_scratch`, which is the newest way to reach it. + debug_assert!( + freelist.runs().iter().all(|r| r.first as u64 + r.len as u64 <= *next_pgno), + "a free run runs past the end of the file: {:?} against {next_pgno}", + freelist.runs() + ); let free_entries = freelist.encode(); // 4. Collect every page that has to reach the disk. A kept chain contributes none. diff --git a/crates/big-pager/src/txn.rs b/crates/big-pager/src/txn.rs index 679334f..040b7f4 100644 --- a/crates/big-pager/src/txn.rs +++ b/crates/big-pager/src/txn.rs @@ -249,7 +249,8 @@ impl<'db, P: PagerMut> WriteTxn<'db, P> { pub fn free(&mut self, pgno: Pgno) { // A page from beyond the tail floor was allocated by this transaction, so it cannot be - // in any committed tree and no reader can be looking at it. + // in any committed tree and no reader can be looking at it. It goes to `scratch`, which + // `alloc` empties first and `commit` gives back - see `Freelist::absorb_scratch`. if pgno as u64 >= self.tail_floor { self.dirty.remove(&pgno); self.scratch.push(pgno); @@ -338,6 +339,15 @@ impl<'db, P: PagerMut> WriteTxn<'db, P> { } pub fn commit(mut self) -> Result { + // **Before anything below allocates.** Nothing in `commit_txn` knows `scratch` exists, + // which is exactly how a transaction whose tree shrank used to lose its own churn into + // `page_count`. It has to happen here rather than deeper in: lowering `next_pgno` after + // a chain page had been taken from the tail would hand that page's number out twice. + self.next_pgno = self.freelist.absorb_scratch( + core::mem::take(&mut self.scratch), + self.next_pgno, + self.txn_id, + ); let txn_id = self.store.commit_txn( self.txn_id, self.base, diff --git a/crates/big-pager/tests/audit.rs b/crates/big-pager/tests/audit.rs new file mode 100644 index 0000000..ec1be8a --- /dev/null +++ b/crates/big-pager/tests/audit.rs @@ -0,0 +1,228 @@ +// Copyright 2026 Bany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Accounting for every page: reachable, free, or neither. +//! +//! **The test that matters most here is the positive control.** A leak counter that returns +//! zero is indistinguishable from one that never looks, so `an_orphaned_page_is_reported` - +//! which deliberately strands a page and demands the counter find it - is what makes every +//! other zero in this file mean something. +//! +//! Everything below drives the pager directly rather than a `Db`, because the trees are the +//! caller's half of the walk. The `Db` half is `crates/big-db/tests/leaks.rs`. + +use big_page::{FragmentKey, LeafBuilder}; +use big_pager::*; + +fn key(field: u32, shard: u64) -> FragmentKey { + FragmentKey::new(1, field, 0, shard) +} + +/// One leaf page, named by a root. Returns its page number. +fn put_fragment(store: &Store

, k: FragmentKey) -> u32 { + let mut w = store.begin_write(); + let p = w.alloc().unwrap(); + w.write(p, LeafBuilder::new().finish(p)).unwrap(); + w.set_root(k, p); + w.commit().unwrap(); + p +} + +/// Runs an audit, marking every root's single leaf page - which is the whole of a tree in +/// these fixtures. The real walk is `Db::audit_pages`. +fn audit(store: &Store

) -> LeakReport { + let mut a = store.begin_audit().expect("the file is readable"); + for root in core::mem::take(&mut a.roots) { + a.mark_tree_page(root, false); + } + for root in core::mem::take(&mut a.snapshot_roots) { + a.mark_tree_page(root, true); + } + a.finish() +} + +// ------------------------------------------------------------------------------------------- +// The bitset +// ------------------------------------------------------------------------------------------- + +#[test] +fn marking_the_same_page_twice_counts_once() { + let mut s = PageSet::with_pages(100); + assert!(s.insert(7)); + assert!(s.insert(7)); + assert_eq!(s.count(), 1); + assert!(s.contains(7)); + assert!(!s.contains(8)); +} + +/// A page past the end of the file is not a member that happens to be absent - it is outside +/// the question, and the caller is told so rather than left to assume it landed. +#[test] +fn a_page_past_the_end_is_refused_rather_than_dropped() { + let mut s = PageSet::with_pages(64); + assert!(!s.insert(64)); + assert!(!s.insert(1_000)); + assert_eq!(s.count(), 0); +} + +/// **The off-by-one that will actually happen.** A file whose page count is not a multiple of +/// 64 has bits in its last word describing pages that do not exist, and a complement that +/// counted them would report leaks which are not pages at all. +#[test] +fn the_complement_ignores_the_bits_past_the_end_of_the_file() { + let pages = 70; // not a multiple of 64 + let mut marks = PageSet::with_pages(pages); + let free = PageSet::with_pages(pages); + for p in 0..pages { + marks.insert(p as u32); + } + assert_eq!(marks.count(), pages); + assert_eq!(marks.count_absent_from_both(&free), 0, "six phantom pages in the last word"); + + marks = PageSet::with_pages(pages); + assert_eq!( + marks.count_absent_from_both(&free), + pages, + "and all of them when nothing is marked" + ); +} + +#[test] +fn the_highest_unaccounted_page_is_the_one_that_pins_the_tail() { + let pages = 200; + let free = PageSet::with_pages(pages); + let mut marks = PageSet::with_pages(pages); + // Everything marked but two holes. The higher one is what a truncation walking down from + // the end of the file meets first, and so the one that matters. + for p in 0..pages { + if p != 40 && p != 150 { + marks.insert(p as u32); + } + } + assert_eq!(marks.count_absent_from_both(&free), 2); + assert_eq!(marks.absent_from_both(&free).next_back(), Some(150)); + assert_eq!(marks.absent_from_both(&free).next(), Some(40)); +} + +// ------------------------------------------------------------------------------------------- +// The audit +// ------------------------------------------------------------------------------------------- + +/// A file that has only ever been written to accounts for every page it has. +#[test] +fn a_fresh_store_accounts_for_every_page() { + let store = Store::init(MemPager::new()).unwrap(); + let r = audit(&store); + assert!(r.is_clean(), "{r:?}"); + assert_eq!(r.leaked, 0); + assert_eq!(r.by_class.meta, 2, "both meta slots, always"); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +#[test] +fn a_store_with_fragments_accounts_for_every_page() { + let store = Store::init(MemPager::new()).unwrap(); + for i in 0..16 { + put_fragment(&store, key(1, i)); + } + let r = audit(&store); + assert_eq!(r.leaked, 0, "{r:?}"); + assert_eq!(r.by_class.trees, 16); + assert_eq!(r.reachable + r.free_total, r.page_count, "{r:?}"); +} + +/// **The positive control.** A page is allocated, written, and then named by nothing: not a +/// root, not a chain, and never freed. Without this test every zero above could be a counter +/// that simply never looks. +#[test] +fn an_orphaned_page_is_reported() { + let store = Store::init(MemPager::new()).unwrap(); + put_fragment(&store, key(1, 0)); + + let stranded = { + let mut w = store.begin_write(); + let p = w.alloc().unwrap(); + w.write(p, LeafBuilder::new().finish(p)).unwrap(); + // No `set_root`, no `free`. The page exists and nothing will ever refer to it again. + w.commit().unwrap(); + p + }; + + let r = audit(&store); + assert_eq!(r.leaked, 1, "{r:?}"); + assert_eq!(r.highest_leaked, Some(stranded)); + assert_eq!(r.leaked_sample, vec![stranded]); + assert!(!r.is_clean()); + assert_eq!(r.reachable + r.free_total + r.leaked, r.page_count, "{r:?}"); +} + +/// Every page a snapshot pins is reachable, even though the current roots have forgotten it. +/// +/// **This is the class no other walk in the tree visits** - `copy_to` skips it deliberately +/// and `scrub` skips it by omission. Without it the audit would report every snapshot-pinned +/// page as leaked, and nobody would find out until a file had a snapshot in it. +#[test] +fn a_tree_only_a_snapshot_still_names_is_reachable() { + let store = Store::init(MemPager::new()).unwrap(); + for i in 0..8 { + put_fragment(&store, key(1, i)); + } + + let mut w = store.begin_write(); + w.create_snapshot(u64::MAX, "before", true); + w.commit().unwrap(); + + // Every fragment goes. Their pages are pending-free and reachable only from the snapshot's + // old roots chain. + for i in 0..8 { + let mut w = store.begin_write(); + let root = w.remove_root(&key(1, i)).expect("still there"); + w.free(root); + w.commit().unwrap(); + } + + let r = audit(&store); + assert_eq!(r.leaked, 0, "a snapshot's trees were counted as leaked: {r:?}"); + assert_eq!(r.by_class.snapshot_trees, 8, "{r:?}"); + assert!(r.by_class.snapshot_chains > 0, "the old roots chain itself: {r:?}"); + assert_eq!(r.double_allocated, 0, "pinned pages are pending-free, never reusable: {r:?}"); +} + +/// An audit reads. It must leave the file at exactly the transaction it found it at. +#[test] +fn an_audit_never_changes_the_file() { + let store = Store::init(MemPager::new()).unwrap(); + for i in 0..8 { + put_fragment(&store, key(1, i)); + } + let before = store.metrics(); + let r = audit(&store); + let after = store.metrics(); + + assert_eq!(before.txn_id, after.txn_id); + assert_eq!(before.page_count, after.page_count); + assert_eq!(before.free_pages_reusable, after.free_pages_reusable); + assert_eq!(r.txn_id, before.txn_id); +} + +/// The reader an audit holds is released when it finishes, or the next truncation would refuse +/// for ever. +#[test] +fn an_audit_releases_its_reader() { + let store = Store::init(MemPager::new()).unwrap(); + put_fragment(&store, key(1, 0)); + let _ = audit(&store); + assert_eq!(store.metrics().live_readers, 0); + assert!(store.truncate_tail().is_ok(), "the audit's reader outlived the audit"); +} diff --git a/crates/big-pager/tests/reclamation.rs b/crates/big-pager/tests/reclamation.rs index 8aff8be..51cdd80 100644 --- a/crates/big-pager/tests/reclamation.rs +++ b/crates/big-pager/tests/reclamation.rs @@ -276,6 +276,58 @@ fn a_file_that_churns_can_be_given_back_to_the_filesystem() { ); } +// ------------------------------------------------------------------------------------------- +// Pages a transaction allocated and then gave up on +// +// A page freed above the tail floor goes to the transaction's scratch list rather than the +// freelist, so `alloc` can hand it straight back. What is left there at commit used to be lost: +// counted in `page_count`, present in the file, and referenced by nothing. Against the freelist +// directly, for the reason at the top of this file. +// ------------------------------------------------------------------------------------------- + +/// A page flush against the end is given back by not growing the file that far, which is +/// better than recording it as free: there is nothing to record and nothing to reuse. +#[test] +fn scratch_against_the_tail_is_given_back_rather_than_freed() { + let mut f = Freelist::default(); + assert_eq!(f.absorb_scratch(vec![10, 11], 12, 5), 10); + assert!(f.is_empty(), "the file simply never grew that far"); +} + +/// A page with something live above it cannot be given back that way, so it is recorded free. +#[test] +fn scratch_below_the_tail_goes_to_the_freelist() { + let mut f = Freelist::default(); + assert_eq!(f.absorb_scratch(vec![10], 12, 5), 12, "page 11 is live, so 10 cannot be dropped"); + assert_eq!(f.runs(), [FreeRun { freed_at: 5, first: 10, len: 1 }]); +} + +/// The giveback stops at the first gap - everything below the gap is recorded instead. +#[test] +fn the_giveback_stops_at_the_first_gap() { + let mut f = Freelist::default(); + assert_eq!(f.absorb_scratch(vec![100, 102], 103, 7), 102); + assert_eq!(f.runs(), [FreeRun { freed_at: 7, first: 100, len: 1 }]); +} + +/// **Stamped with the committing transaction, so it is pending for the whole of that commit.** +/// A lower stamp would make the page allocatable while the freelist's own pages are still being +/// chosen, which is the loop whose termination argument assumes the entry count only falls. +#[test] +fn an_absorbed_page_is_not_reusable_inside_its_own_commit() { + let mut f = Freelist::default(); + f.absorb_scratch(vec![10], 12, 5); + assert_eq!(f.alloc(4), None, "the horizon is below this transaction"); + assert_eq!(f.alloc(5), Some(10), "and it is reusable once the horizon reaches it"); +} + +#[test] +fn absorbing_nothing_changes_nothing() { + let mut f = Freelist::default(); + assert_eq!(f.absorb_scratch(Vec::new(), 12, 5), 12); + assert!(f.is_empty()); +} + #[test] fn readers_are_refcounted_not_flagged() { let store = Store::init(MemPager::new()).unwrap(); diff --git a/crates/big-pager/tests/store.rs b/crates/big-pager/tests/store.rs index e9a7bc6..8c89fb1 100644 --- a/crates/big-pager/tests/store.rs +++ b/crates/big-pager/tests/store.rs @@ -266,6 +266,61 @@ fn backend_is_swappable() { } } +/// **A transaction that gives its own pages back must leave the file the size it would have +/// been.** Two stores, the same committed content; one of them additionally takes five pages +/// from the tail and frees them all inside the same transaction. Those pages are in no tree, in +/// no chain, and no reader ever saw them - a file that ends up larger for them has kept pages +/// nothing will ever read and nothing will ever hand out again. +/// +/// A difference test rather than an absolute one, so it stays true when the machinery around it +/// changes how many pages a commit needs. +#[test] +fn a_transaction_that_abandons_its_own_pages_leaves_no_trace() { + let plain = Store::init(MemPager::new()).unwrap(); + put_fragment(&plain, key(1, 0)); + + let churned = Store::init(MemPager::new()).unwrap(); + put_fragment(&churned, key(1, 0)); + let mut w = churned.begin_write(); + let pages: Vec = (0..5).map(|_| w.alloc().unwrap()).collect(); + for p in pages { + w.free(p); + } + w.commit().unwrap(); + // The plain store commits too, so both have run the same number of transactions. + plain.begin_write().commit().unwrap(); + + assert_eq!( + churned.pager().page_count(), + plain.pager().page_count(), + "five pages allocated and freed in one transaction were kept" + ); +} + +/// The plateau below, but with a transaction that churns. **This is the one the leak broke:** +/// an empty commit has nothing in its scratch list, so it plateaus either way. +#[test] +fn repeated_churning_commits_do_not_grow_the_file_without_bound() { + let store = Store::init(MemPager::new()).unwrap(); + put_fragment(&store, key(1, 0)); + let churn = || { + let mut w = store.begin_write(); + let pages: Vec = (0..3).map(|_| w.alloc().unwrap()).collect(); + for p in pages { + w.free(p); + } + w.commit().unwrap(); + }; + for _ in 0..50 { + churn(); + } + let settled = store.pager().page_count(); + for _ in 0..200 { + churn(); + } + assert_eq!(store.pager().page_count(), settled, "page count must plateau under churn"); +} + /// Freelist pages always come from the file tail, so a quiet loop of commits must still /// reach a steady state rather than growing the file forever. #[test] From 771568abd2dae37bcf4da7c2df92bc89d9b4e118 Mon Sep 17 00:00:00 2001 From: 9bany Date: Sat, 5 Sep 2026 10:40:43 +0700 Subject: [PATCH 5/6] docs(deploy): say which self-management flags the two-node demo can use Three shipped behind their own flags and none of them reached the demo, so the file described a cluster one release behind itself. `--reclaim` is on for both nodes. It is the one that needs nothing from the cluster - no peer asked, nothing proposed - and the comment says what it does not do as well: it gives back the free run at the end of the file, so a database that has churned in the middle keeps those holes until an offline `big compact`. Its floor is 4096 pages with a quarter reusable, which a demo this small never reaches, and saying so is cheaper than someone wondering why the counter never moves. `--balance` and `--elect-schema-leader` stay off, each with its reason where an operator will look for it. The second reason is the one worth writing down: **with two voters it could not fire even if it were on.** Handing the row-key namespace on is a decision the agreement commits, a majority of two is two, and the silence it waits for is the same node being gone. It wants the third node that replication wants. --- deploy/cluster/docker-compose.yml | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/deploy/cluster/docker-compose.yml b/deploy/cluster/docker-compose.yml index a031114..dc88ba0 100644 --- a/deploy/cluster/docker-compose.yml +++ b/deploy/cluster/docker-compose.yml @@ -24,6 +24,27 @@ # that is up, it cannot pick data that is not there. Failing over needs a copy of a range, and a # copy needs a third node to settle the vote - see docs/clustering.md, "Replication, and failing # over", and deploy/readme.md, "What a joining node needs first", for what adds one. +# +# **Three things a node can now do for itself, and only one of them is on here.** Each is a flag +# and each is off by default, for one reason: a deployment that reshapes itself unasked is one +# whose shape an operator cannot predict. +# +# --reclaim on below. Purely local - it asks no peer and proposes nothing, so it +# runs on every node, clustered or not, leader or not. See the flag. +# --balance off. It works, and it moves the facts rather than only the map: a +# range handed over is seeded, caught up, cut over behind a write +# barrier, checked by digest, and only then dropped from the source. +# Off because the two ranges here are the thing this file is +# explaining, and a cluster that redraws them mid-demo is explaining +# something else. It also needs both nodes up to decide anything, for +# the reason below. +# --elect-schema-leader off, and here it would be **inert rather than merely cautious**. It +# hands the row-key namespace on after fifteen seconds of silence from +# whoever holds it, and handing it on is a decision the agreement has +# to commit. A majority of two is two - so at the moment the schema +# leader is the node that went away, there is no majority left to +# commit anything with. This flag wants the same third node that +# replication wants. # Everything the two nodes share. Not the volumes: a merge key replaces a list rather than # extending it, so a `volumes` here would be silently overridden by each service's own and read @@ -88,6 +109,16 @@ services: # ceiling is a daemon one expensive query can hold a worker on for as long as it likes. - --query-timeout - "30" + # Trailing free pages go back to the filesystem while this node serves. Local, and gated + # on nothing but this switch: no peer is asked and nothing is proposed, so it does not + # care that this is a cluster. It waits for the file to reach 4096 pages with a quarter of + # it reusable, which a demo this small never reaches - it is here because this file is + # also a template. **What it gives back is the free run at the end of the file and nothing + # else**: a database that has churned in the middle keeps those holes until an offline + # `big compact` rewrites it, because moving a live page means rewriting whoever points at + # it. `big leaks ` prints the accounting, offline. Watch big_pages_reclaimed_total + # and big_reclaim_blocked_total; the second one climbing means readers, not a fault. + - --reclaim # **The same certificate again, and this is the pair that makes the cluster work.** # `--peer-cert` is the identity this node presents when it *dials* another; `--tls-cert` is # what its own listener presents when it is dialled. A node given only the first speaks TLS @@ -137,6 +168,16 @@ services: # ceiling is a daemon one expensive query can hold a worker on for as long as it likes. - --query-timeout - "30" + # Trailing free pages go back to the filesystem while this node serves. Local, and gated + # on nothing but this switch: no peer is asked and nothing is proposed, so it does not + # care that this is a cluster. It waits for the file to reach 4096 pages with a quarter of + # it reusable, which a demo this small never reaches - it is here because this file is + # also a template. **What it gives back is the free run at the end of the file and nothing + # else**: a database that has churned in the middle keeps those holes until an offline + # `big compact` rewrites it, because moving a live page means rewriting whoever points at + # it. `big leaks ` prints the accounting, offline. Watch big_pages_reclaimed_total + # and big_reclaim_blocked_total; the second one climbing means readers, not a fault. + - --reclaim # **The same certificate again, and this is the pair that makes the cluster work.** # `--peer-cert` is the identity this node presents when it *dials* another; `--tls-cert` is # what its own listener presents when it is dialled. A node given only the first speaks TLS From b74134b6982bd8ac80e2c814861e0fc6f52db960 Mon Sep 17 00:00:00 2001 From: 9bany Date: Sat, 5 Sep 2026 11:20:56 +0700 Subject: [PATCH 6/6] feat(deploy): a Kubernetes deployment that grows with `kubectl scale` The obvious shape - one StatefulSet per node, because a node here is not interchangeable with the next one - was written first and thrown away. It is the correct argument and the wrong conclusion: what it produces is a deployment where growing the cluster means writing YAML, and the cluster has been able to admit a learner and give it work by itself since `--balance`. A knob that looks like scaling and produces an idle stranger is worse than no knob; a shape that has no knob at all is worse still. So: one StatefulSet, pod names as node names, and the four things a joining node needs split by who can actually supply them. **Only two of them are Kubernetes's.** A certificate is issued in advance - names are cheap, because a certificate grants nothing until the roster accepts it and the roster follows the agreement rather than a file. The cluster file is written per pod by an initContainer. The other two are the cluster's own: `add-node` registers the pod as a learner, and the balancer on the leader admits it and cuts the tail of the shard space over to it. **`add-node` runs before the daemon, and that ordering is the whole reason there is an initContainer rather than a sidecar.** A node that starts while the cluster has never heard of it is a voter in a cluster of its own imagining, and it stands for election once per timeout, raising the term on every node that hears it. Registered first, it is reached as a learner, and a learner does not stand. Three lists, three rules, and confusing them is the mistake this deployment makes easy, so they are in one file with the difference at the top. `seed.toml` is a membership and must be exact: a node with no log counts a majority out of it, so naming a pod that is not running is naming a vote that never arrives. `upstreams.toml` is an address space and deliberately names pods that do not exist - the proxy polls, finds them unreachable and leaves them out of rotation, which is what lets a scaled-up node reach a client with no proxy restart. The certificates are the third. Three tests pin all of it, including the file `join.sh` generates: a file the daemon refuses is a pod that crash-loops with no clue pointing back at a shell script inside a ConfigMap. Three nodes and not the demo's two, because two is the size at which nothing works: a majority of two is two, so neither a failover nor a schema-leader handover can be committed at the moment one node is the one that went away. `--balance` and `--elect-schema-leader` are on here for the same reason they are off in `cluster/` - there, the two ranges are the thing being explained; here the shape is meant to change. `certs.sh` and `users.sh` grow `NODES=` and `SECRETS=` overrides, so this deployment gets a CA and a users file of its own rather than a second copy of either script: two deployments sharing one CA are two deployments that can impersonate each other. Scaling in is not the mirror image and is not automated: drain, wait, remove, then scale. `cluster remove` refuses a node that still holds a range, and that refusal is the point. --- .gitignore | 1 + README.md | 2 +- crates/big-proxy/tests/config.rs | 90 ++++++++++ deploy/cluster/certs.sh | 15 +- deploy/cluster/users.sh | 18 +- deploy/k8s/00-namespace.yaml | 17 ++ deploy/k8s/10-config.yaml | 206 +++++++++++++++++++++ deploy/k8s/20-nodes.yaml | 295 +++++++++++++++++++++++++++++++ deploy/k8s/30-proxy.yaml | 166 +++++++++++++++++ deploy/k8s/kustomization.yaml | 19 ++ deploy/k8s/readme.md | 277 +++++++++++++++++++++++++++++ deploy/k8s/secrets.sh | 126 +++++++++++++ deploy/readme.md | 10 +- 13 files changed, 1229 insertions(+), 13 deletions(-) create mode 100644 deploy/k8s/00-namespace.yaml create mode 100644 deploy/k8s/10-config.yaml create mode 100644 deploy/k8s/20-nodes.yaml create mode 100644 deploy/k8s/30-proxy.yaml create mode 100644 deploy/k8s/kustomization.yaml create mode 100644 deploy/k8s/readme.md create mode 100755 deploy/k8s/secrets.sh diff --git a/.gitignore b/.gitignore index bea80fb..7a2b6b0 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ archived # Generated by deploy/cluster/tokens.sh. A credential that reaches a repository is not a # credential. deploy/cluster/secrets +deploy/k8s/secrets deploy/single/secrets # `make start`: the local database, its log and its pid. A playground, not a fixture. diff --git a/README.md b/README.md index e3c0493..0627071 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ crates/big-bin/ the `big` and `bigctl` binaries crates/big-http/ the server crates/big-embed/ the library API contrib/ written against the HTTP surface, as an outside user would -deploy/ single-node and cluster deployments +deploy/ single-node, compose cluster and Kubernetes deployments examples/ runnable docker compose stacks bench/ benchmarks; excluded from the workspace test run fuzz/ cargo-fuzz targets, nightly only diff --git a/crates/big-proxy/tests/config.rs b/crates/big-proxy/tests/config.rs index 25fe4f8..476bece 100644 --- a/crates/big-proxy/tests/config.rs +++ b/crates/big-proxy/tests/config.rs @@ -101,3 +101,93 @@ fn neither_reader_accepts_a_file_with_no_nodes() { assert!(ClusterFile::parse(empty).is_err()); assert!(parse_cluster(empty, "empty").is_err()); } + +// ------------------------------------------------------------------------------------------- +// The Kubernetes manifests +// ------------------------------------------------------------------------------------------- +// +// `deploy/k8s` ships two TOML documents inside one ConfigMap, and they are not +// interchangeable - which is exactly the mistake that is easy to make while editing YAML. The +// tests below pin the difference: +// +// - `seed.toml` is a **membership**: a node with no log counts a majority out of it, so it must +// satisfy the daemon's reader, ranges and all. +// - `upstreams.toml` is an **address space**: it deliberately names pods that do not exist yet, +// so that `kubectl scale` reaches a client with no proxy restart. Only this crate's reader +// ever sees it, and it must accept it. +// +// A third file exists only at runtime: `join.sh` writes the cluster file a joining pod starts +// with, by appending a `replica` entry per name up to its own ordinal. That append is +// reproduced here, because a file the daemon refuses is a pod that crash-loops with no clue +// pointing back at a shell script in a ConfigMap. + +/// One `key: |` block scalar out of a ConfigMap, dedented. +/// +/// A parser rather than a YAML dependency: what is needed is one block, the indentation is +/// fixed at four spaces by the file itself, and a test that pulled in a YAML crate to read two +/// strings would be a dependency the shipped binary is measured against. +fn block(yaml: &str, key: &str) -> String { + let opener = format!(" {key}: |"); + let start = yaml.find(&opener).unwrap_or_else(|| panic!("no `{key}:` block in the manifest")); + let body = &yaml[start + opener.len()..]; + let mut out = String::new(); + for line in body.lines().skip(1) { + if !line.is_empty() && !line.starts_with(" ") { + break; + } + out.push_str(line.strip_prefix(" ").unwrap_or("")); + out.push('\n'); + } + out +} + +fn manifest() -> String { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../deploy/k8s/10-config.yaml"); + std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display())) +} + +#[test] +fn the_k8s_seed_is_a_membership_both_readers_accept() { + let seed = block(&manifest(), "seed.toml"); + let daemon = ClusterFile::parse(&seed).expect("the seed is what a founding node starts from"); + let proxy = parse_cluster(&seed, "seed").expect("and the proxy could read it too"); + + let names: Vec<&str> = daemon.nodes().iter().map(|n| n.name.as_str()).collect(); + assert_eq!(names, ["big-0", "big-1", "big-2"], "three, because a majority of two is two"); + assert_eq!(proxy.len(), 3); + assert_eq!(proxy[0].addr, "big-0.nodes:7654", "the address is the headless Service's"); + assert_ne!(proxy[0].name, proxy[0].addr, "the name is what the certificate carries"); +} + +#[test] +fn the_k8s_upstream_list_may_name_pods_that_do_not_exist() { + let upstreams = block(&manifest(), "upstreams.toml"); + let proxy = parse_cluster(&upstreams, "upstreams").expect("a front door does not vote"); + assert!(proxy.len() > 3, "the ceiling is above the founding size, or scaling needs a restart"); + + // And the daemon must refuse it, which is why it is a separate file: every entry past the + // third has no range, so this is not a shard map and was never meant to be seeded from. + assert!( + ClusterFile::parse(&upstreams).is_err(), + "if the daemon accepts this, somebody has given the upstream list shard ranges - and a \ + node seeded from it would count a majority out of pods that do not exist" + ); +} + +#[test] +fn a_joining_pods_generated_file_is_one_the_daemon_accepts() { + // What `join.sh` writes for `big-4`: the seed, then every name up to its own ordinal that + // the seed does not already have, as a replica of the first node. + let mut file = block(&manifest(), "seed.toml"); + for i in 3..=4 { + file.push_str(&format!( + "\n[[node]]\nname = \"big-{i}\"\naddr = \"big-{i}.nodes:7654\"\nreplica = \"big-0\"\n" + )); + } + + let daemon = ClusterFile::parse(&file).expect("a joining node must be able to start"); + assert_eq!(daemon.nodes().len(), 5); + // The roster is the point: the leader that dials this node may be any of them. + let names: Vec<&str> = daemon.nodes().iter().map(|n| n.name.as_str()).collect(); + assert!(names.contains(&"big-0") && names.contains(&"big-4")); +} diff --git a/deploy/cluster/certs.sh b/deploy/cluster/certs.sh index c837fbf..f38e126 100755 --- a/deploy/cluster/certs.sh +++ b/deploy/cluster/certs.sh @@ -11,12 +11,19 @@ set -eu cd "$(dirname "$0")" -nodes="a b" +# **The names, and where they are written**, both overridable so that a second deployment can +# have a CA of its own without a second copy of this script. `deploy/k8s` uses both: its nodes +# are a StatefulSet's pods, so they are called `big-0`, `big-1`, and it keeps its own `secrets/` +# because two deployments sharing one CA is two deployments that can impersonate each other. +# +# NODES="big-0 big-1 big-2" SECRETS=../k8s/secrets ./certs.sh +nodes=${NODES:-"a b"} +out=${SECRETS:-secrets} days=${DAYS:-365} -mkdir -p secrets -chmod 700 secrets -cd secrets +mkdir -p "$out" +chmod 700 "$out" +cd "$out" # **With a name, this issues one more certificate and leaves everything else alone.** A node that # joins a running cluster needs one, and the alternative - re-running the whole script - would diff --git a/deploy/cluster/users.sh b/deploy/cluster/users.sh index 3a510b2..5a33586 100755 --- a/deploy/cluster/users.sh +++ b/deploy/cluster/users.sh @@ -30,11 +30,15 @@ command -v "$big" >/dev/null || { exit 1 } -mkdir -p secrets -chmod 700 secrets +# Where to write it. `deploy/k8s` points this at its own directory, because a users file is a +# deployment's, not a machine's - see certs.sh for the same override and the same reason. +out=${SECRETS:-secrets} -if [ -f secrets/users ]; then - echo "secrets/users already exists; delete it first if you mean to rotate" >&2 +mkdir -p "$out" +chmod 700 "$out" + +if [ -f "$out/users" ]; then + echo "$out/users already exists; delete it first if you mean to rotate" >&2 exit 1 fi @@ -44,15 +48,15 @@ reader=$(random) # Piped rather than typed: `big passwd` reads one line from standard input when it is not a # terminal, which is the only non-tty path and exists for exactly this. It is never a flag, # because an argument is visible in `ps`. -printf '%s\n' "$admin" | "$big" passwd secrets/users set ops --role superuser +printf '%s\n' "$admin" | "$big" passwd "$out/users" set ops --role superuser # **Named before it exists**, and that is the right order: a role is resolved by name on every # request, so creating `dashboard` in the catalog later makes this credential work without a # restart. What needs a restart is changing the *name* in this file - it is read once, at # startup - which is the trap worth knowing about and the reason the name is chosen now. -printf '%s\n' "$reader" | "$big" passwd secrets/users set dashboard --role dashboard +printf '%s\n' "$reader" | "$big" passwd "$out/users" set dashboard --role dashboard echo -echo "wrote secrets/users (mode 600)" +echo "wrote $out/users (mode 600)" echo " ops: $admin (superuser)" echo " dashboard: $reader (role \`dashboard\`, which does not exist yet)" echo diff --git a/deploy/k8s/00-namespace.yaml b/deploy/k8s/00-namespace.yaml new file mode 100644 index 0000000..58c950e --- /dev/null +++ b/deploy/k8s/00-namespace.yaml @@ -0,0 +1,17 @@ +# One namespace, because every name in here is a DNS name the cluster depends on. +# +# `cluster.toml` says a node is reached at `a:7654`, and that short name resolves only from +# inside the namespace the Service `a` lives in - the resolver's search path is +# `.svc.cluster.local` first. Two copies of this deployment in one namespace would be +# two nodes called `a`; two namespaces is two clusters that cannot see each other, which is what +# you want and is free. +apiVersion: v1 +kind: Namespace +metadata: + name: big + labels: + # The restricted profile is what these manifests are written against: no root, no added + # capabilities, seccomp on. The entrypoint is happy unprivileged - see the readme, "Why + # nothing here runs as root and the Secret is mode 0440". + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest diff --git a/deploy/k8s/10-config.yaml b/deploy/k8s/10-config.yaml new file mode 100644 index 0000000..6feaf33 --- /dev/null +++ b/deploy/k8s/10-config.yaml @@ -0,0 +1,206 @@ +# Three lists, and they follow three different rules. Getting them confused is the whole +# difficulty of running this on Kubernetes, so they are in one file with the difference written +# down. +# +# **`seed.toml` is the membership, and it must be exact.** A node with no log seeds its agreement +# from this file and counts a majority out of it, so naming a node that is not running is naming +# a vote that never arrives: three entries with two pods up is a cluster that cannot elect +# anybody. It names the *founding* nodes and is not edited when the cluster grows - growth is a +# committed decision, and `bigctl cluster topology` is where the current membership is. +# +# **`upstreams.toml` is an address space, and it may name nodes that do not exist.** The proxy +# does not vote; a name it cannot reach is one it polls, marks unreachable and leaves out of +# rotation. So it is written out to the ceiling once, and a node that appears later is picked up +# within a health interval with no restart and no edit. Raise the ceiling here and in +# `secrets.sh` - those two lists are the same list. +# +# **`join.sh` writes the third one**, per pod, at startup: the file *that* node begins with. See +# the script. +apiVersion: v1 +kind: ConfigMap +metadata: + name: big-seed +data: + seed.toml: | + # The cluster as it is founded, and only that. `kubectl scale` does not edit this file; a + # node that joins afterwards is registered with `cluster add-node` and given a range by the + # balancer, and the agreement is what says who is in the cluster from then on. + # + # Three nodes and not two, because two is the size at which nothing works: a majority of two + # is two, so a cluster of two can neither fail a range over nor hand the schema-key + # namespace on - both are decisions a majority has to commit while one node is the one that + # went away. `deploy/cluster` runs two because compose runs on one machine and the second + # node is there to show the shape. This is not that. + cluster_id = "big-k8s" + + schema_leader = "big-0" + peer_ca_file = "/run/big/peer-ca.pem" + + # The names are the pods'; the addresses are the headless Service's. They differ, and the + # daemon is built for them differing: a certificate is issued to a *name*, and + # `ClientWire::connect` is handed `peer.name` as the TLS server name rather than the host it + # dialled. So a node keeps its identity wherever it is reached. + [[node]] + name = "big-0" + addr = "big-0.nodes:7654" + shards = "0..64" + + [[node]] + name = "big-1" + addr = "big-1.nodes:7654" + shards = "64..128" + + [[node]] + name = "big-2" + addr = "big-2.nodes:7654" + shards = "128.." + + upstreams.toml: | + # What the proxy may forward to. It reads `name` and `addr` and discards the rest - it does + # not route by key - so there are no ranges here and no ceiling on how wrong they could be. + # + # **Eight names for three nodes, on purpose.** A name whose pod does not exist is polled, + # found unreachable and left out of rotation; a name that appears is in rotation within a + # health interval. That is what makes `kubectl scale big --replicas=5` reach the client + # without a proxy restart. Nothing here grants anything: a certificate is what a node + # authenticates with, and the roster that accepts one follows the agreement. + cluster_id = "big-k8s" + + [[node]] + name = "big-0" + addr = "big-0.nodes:7654" + [[node]] + name = "big-1" + addr = "big-1.nodes:7654" + [[node]] + name = "big-2" + addr = "big-2.nodes:7654" + [[node]] + name = "big-3" + addr = "big-3.nodes:7654" + [[node]] + name = "big-4" + addr = "big-4.nodes:7654" + [[node]] + name = "big-5" + addr = "big-5.nodes:7654" + [[node]] + name = "big-6" + addr = "big-6.nodes:7654" + [[node]] + name = "big-7" + addr = "big-7.nodes:7654" +--- +# What each pod runs before its daemon starts: it writes the cluster file *that* node begins +# with, and registers the node with the cluster it is joining. +# +# **Both halves exist because of one rule in the daemon**: a node's listener accepts a peer whose +# certificate names somebody on its roster, and the roster it starts with comes from its own +# file. A node whose file names only itself refuses every peer that dials it, has none of its own +# to dial, and sits there healthy and alone. Once it is in touch the roster follows the +# agreement - `Controller::follow_roster` - so this is only about the first second. +# +# **It is a shell script and not an operator**, and the ceiling on what it will do is: write a +# file, and make one call. It does not choose ranges, does not admit anybody and does not scale +# anything. The balancer does those, on the leader, from facts it gathers itself. +apiVersion: v1 +kind: ConfigMap +metadata: + name: big-join +data: + join.sh: | + #!/bin/sh + # Runs as an initContainer. Never fails the pod: a cluster that cannot be reached yet is + # what a bootstrap looks like, and the daemon behind this is what makes it reachable. + set -eu + + me=${POD_NAME:?POD_NAME is not set} + svc=${PEER_SERVICE:?PEER_SERVICE is not set} + seed=/etc/big/seed/seed.toml + gen=/etc/big/gen/cluster.toml + ca=/etc/big/secrets/peer-ca.pem + cred=/etc/big/join/credentials + + # The `name = "..."` of every [[node]], in file order. Anchored past leading space so a + # commented-out entry is not read as one. + names() { sed -n 's/^[[:space:]]*name[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$1"; } + has() { names "$2" | grep -qx "$1"; } + + cp "$seed" "$gen" + + if has "$me" "$gen"; then + echo "join: $me is a founding node; starting from the seed unchanged" + else + # **Every name up to this pod's own ordinal, as a replica of the first.** + # + # Not just this node: the peer that dials a joining node is the agreement's *leader*, + # and by the time a cluster has grown the leader may be a node the seed never mentioned. + # A roster that named only the founders and this pod would refuse it and the join would + # hang for no visible reason. + # + # `replica` and not `shards` because it is the shape that parses without claiming a + # range anybody else owns - and a claim would be a lie this node might act on if it were + # ever alone. The agreement overwrites all of it within the second. Names of pods that + # do not exist, or that were removed, cost nothing: a roster is a list of names a + # certificate may claim, not a list of votes. + first=$(names "$gen" | head -1) + ord=${me##*-} + stem=${me%-*} + i=0 + while [ "$i" -le "$ord" ]; do + n="$stem-$i" + i=$((i + 1)) + if has "$n" "$gen"; then continue; fi + printf '\n[[node]]\nname = "%s"\naddr = "%s.%s:7654"\nreplica = "%s"\n' \ + "$n" "$n" "$svc" "$first" >> "$gen" + done + echo "join: wrote a cluster file naming $(names "$gen" | tr '\n' ' ')" + fi + + # A founding node is in the seed membership already; there is nobody to ask and nothing to + # add. + if has "$me" "$seed"; then + exit 0 + fi + + # **Registering is a separate act from being reachable, and it is meant to be.** A node that + # dials in is not thereby a member: `add-node` is a decision an existing cluster commits, + # and the credential below is what says somebody authorised it. Without one this pod still + # starts - it just sits outside the cluster until an operator runs the command printed here. + if [ ! -r "$cred" ]; then + echo "join: no credential mounted, so $me will not register itself. Run:" >&2 + echo "join: bigctl cluster add-node $me $me.$svc:7654" >&2 + exit 0 + fi + + # `bigctl` holds a credentials file to the same rule the daemon holds a users file to, and a + # Secret cannot be both readable here and mode 600 - see the readme. A private copy can. + install -m 600 "$cred" /run/join/credentials + + # **Before the daemon, not after**, which is the ordering that matters: a node that starts + # while the cluster has never heard of it is a voter in a cluster of its own imagining, and + # it will stand for election once per timeout until somebody tells it otherwise. Registered + # first, the leader reaches it as a learner - which does not vote and does not stand. + # + # Only the leader can commit this, and it does not forward, so this asks each node in turn. + for peer in $(names "$gen"); do + if [ "$peer" = "$me" ]; then continue; fi + if out=$(bigctl --addr "https://$peer.$svc:7654" --ca-file "$ca" \ + --credentials-file /run/join/credentials \ + cluster add-node "$me" "$me.$svc:7654" 2>&1); then + echo "join: $peer took the registration" + exit 0 + fi + # Already a member: a pod that restarted, or a second pass. Nothing to do and not a + # failure. + case $out in + *"already in this cluster"*) + echo "join: $me is already a member" + exit 0 + ;; + esac + done + + echo "join: nobody took the registration; starting anyway. The cluster may still be" >&2 + echo "join: bootstrapping - 'bigctl cluster topology' says where it stands." >&2 + exit 0 diff --git a/deploy/k8s/20-nodes.yaml b/deploy/k8s/20-nodes.yaml new file mode 100644 index 0000000..8fa4d34 --- /dev/null +++ b/deploy/k8s/20-nodes.yaml @@ -0,0 +1,295 @@ +# The nodes: one StatefulSet, and `kubectl scale` is how it grows. +# +# **What makes that possible is not this file.** A node that appears has to be given a +# certificate, a cluster file that names the cluster, a membership entry, and a range - four +# things, and only the first two are Kubernetes's. The daemon does the other two by itself: +# `add-node` registers the pod as a **learner**, which holds nothing and does not vote, and the +# balancer on the agreement's leader admits it once it has caught up and then cuts the tail of +# the shard space over to it - `Action::Admit` then `Action::SplitTail`, one decision per pass, +# each one committed. That is `--balance` below, and it is why there is no YAML to write per +# node. +# +# So the four become: a pre-issued certificate (`secrets.sh`, and names are cheap), a file the +# initContainer writes (`join.sh`), and two decisions the cluster makes about itself. +# +# **Scaling in is not the mirror image**, and deliberately so. Deleting a pod that holds a range +# takes that range offline, and `cluster remove` refuses a node that still holds one. Drain it +# first - see the readme. +apiVersion: v1 +kind: Service +metadata: + name: nodes + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node +spec: + # Headless, so `big-3.nodes` is the pod itself: peer traffic goes straight to the daemon, and + # a StatefulSet's pod gets the stable name its certificate was issued for. + clusterIP: None + # **The one Kubernetes-specific decision here.** Endpoints are the ready ones by default, so a + # node that has not passed its probe would have no address - and a peer that cannot reach it + # cannot help it settle the election that would make it ready. This is the flag that is not + # that deadlock, and it is what a joining pod is dialled at before it is anything. + publishNotReadyAddresses: true + selector: + app.kubernetes.io/name: big + app.kubernetes.io/component: node + ports: + - name: big + port: 7654 + targetPort: big +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: big + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node +spec: + serviceName: nodes + # **Three, and two would be worse than one.** A majority of two is two, so a two-node cluster + # can neither fail a range over nor hand the schema-key namespace on: both are decisions a + # majority has to commit at the moment one node is the one that went away. This is the + # smallest size at which the two flags below mean anything. + # + # `seed.toml` names exactly these three. Everything above three is a join, and joins are not + # in a file. + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node + # The default, written out because it looks like a trap and is not one: ordered start waits + # for each pod to be *ready*, and readiness here means "the listener answers" rather than "the + # range is being served". A bootstrap therefore does not deadlock waiting for a quorum that + # needs the next pod. See the readme on what a probe here can and cannot say. + podManagementPolicy: OrderedReady + # Deleting a StatefulSet must not be a way to delete a database. + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + template: + metadata: + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node + spec: + terminationGracePeriodSeconds: 2 + affinity: + podAntiAffinity: + # Preferred and not required: on a one-machine cluster - kind, minikube, what this is + # first run on - `required` would leave two of the three Pending forever. What it buys + # on a real cluster is that one machine failing is one range down rather than three. + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: big + app.kubernetes.io/component: node + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + # What makes the Secret's `0440` readable: its files are owned `root:`. It is + # also what makes the volume writable without the image chowning anything. + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + initContainers: + # Writes the cluster file this pod starts with, and registers the pod if it is new. + # Never fails the pod: a cluster that cannot be reached yet is what a bootstrap looks + # like. See the `big-join` ConfigMap for the whole of it. + - name: join + image: big:latest + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "/etc/big/join-script/join.sh"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: PEER_SERVICE + value: nodes + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + resources: + requests: + cpu: 10m + memory: 32Mi + volumeMounts: + - name: gen + mountPath: /etc/big/gen + - name: seed + mountPath: /etc/big/seed + readOnly: true + - name: join-script + mountPath: /etc/big/join-script + readOnly: true + - name: secrets + mountPath: /etc/big/secrets + readOnly: true + - name: join-credential + mountPath: /etc/big/join + readOnly: true + - name: join-run + mountPath: /run/join + containers: + - name: big + image: big:latest + imagePullPolicy: IfNotPresent + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + args: + - /data/big.db + - 0.0.0.0:7654 + # The file the initContainer wrote, not the ConfigMap. A founding node's copy is the + # seed unchanged; a joining node's names the cluster it is joining as well. + - --cluster + - /etc/big/gen/cluster.toml + # Which entry in that file this daemon is. Kubernetes expands `$(POD_NAME)` from the + # environment above, so the pod's ordinal is the node's name and the certificate + # issued to it is the one the entrypoint stages - it reads `--node` out of this list + # rather than out of a second variable that could disagree with it. + - --node + - $(POD_NAME) + - --users + - /run/big/users + - --peer-cert + - /run/big/peer.pem + - --peer-key + - /run/big/peer.key + - --tls-cert + - /run/big/peer.pem + - --tls-key + - /run/big/peer.key + - --query-timeout + - "30" + # **On here, where the compose demo has it off.** This is the flag that makes + # `kubectl scale` a scale: on the agreement's leader it admits a learner that has + # caught up, gives a node with nothing the tail of the shard space - a split that + # moves no bytes - and moves a range off a node an operator is draining. One + # decision per pass, each committed, and it does nothing at all on a node that is + # not the leader. + # + # `deploy/cluster` leaves it off because two nodes and two ranges are the thing that + # file is explaining, and a cluster that redraws them mid-demo is explaining + # something else. Here the shape is meant to change. + - --balance + # Hands the row-key namespace on after fifteen seconds of silence from whoever holds + # it. Inert with two nodes and real with three: handing it on is a decision the + # agreement has to commit, and the moment it is needed is the moment its holder is + # the node that went away. + - --elect-schema-leader + # Purely local: it asks no peer and proposes nothing. Gives the free run at the end + # of the file back to the filesystem while this node serves. + - --reclaim + - --backup-dir + - /data + ports: + - name: big + containerPort: 7654 + # `scheme: HTTPS` with no certificate check, which is what is wanted: the kubelet + # dials the pod's IP and the certificate names `big-3`. The question is "can this + # process answer", not "is this the right process", and it never leaves the machine to + # ask it. The compose healthcheck passes `-k` for the same reason. + livenessProbe: + httpGet: { path: /health, port: big, scheme: HTTPS } + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + # **"Answering", not "serving", and it cannot be made to say more.** `/ready` is always + # `200` - `serving` is a field in the body - and an httpGet probe reads the status line + # only. The proxy is what reads the body. See 30-proxy.yaml. + readinessProbe: + httpGet: { path: /ready, port: big, scheme: HTTPS } + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + # A request, so the scheduler knows what it is placing. **No memory limit**: reads come + # out of the mapping, so most of what this process appears to use is page cache the + # kernel will give back, and a hard ceiling turns that into an OOM kill under load. + # Set one if you have measured your workload, not because a template had one. + resources: + requests: + cpu: 100m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + volumeMounts: + - name: data + mountPath: /data + # Where the entrypoint stages the credentials at mode 600. Must be writable, and + # must not outlive the pod: what is in it is a private copy of a private key. + - name: run + mountPath: /run/big + - name: gen + mountPath: /etc/big/gen + readOnly: true + - name: secrets + mountPath: /etc/big/secrets + readOnly: true + volumes: + - name: run + emptyDir: {} + - name: join-run + emptyDir: {} + # Written by the initContainer, read by the daemon. + - name: gen + emptyDir: {} + - name: seed + configMap: + name: big-seed + items: + - key: seed.toml + path: seed.toml + - name: join-script + configMap: + name: big-join + # **One Secret for every pod, and that is the price of one StatefulSet.** A pod stages + # only the certificate `--node` names, so no daemon presents anybody else's - but every + # pod could read every key, which per-node certificates were meant to avoid. Per-node + # Secrets need per-node workloads, which is the shape that cannot be scaled. If that + # trade is the wrong way round for you, issue per-pod certificates with cert-manager's + # CSI driver and mount that instead; nothing above changes. + # + # `0440` and not `0400`: the files are owned `root:` and the daemon is not + # root, so `0400` would be unreadable and the entrypoint would stage nothing - which + # presents as `cannot read /run/big/users`, not as a permission error on the mount. + # `0440` is also why the staging is not optional: `big serve` refuses a users file or a + # key that is group-readable, and no `defaultMode` is both readable here and mode 600. + - name: secrets + secret: + secretName: big-nodes + defaultMode: 0440 + # The credential the initContainer registers with, and the only optional volume here: a + # cluster with no `big-ops` Secret still starts, and prints the one `add-node` command an + # operator then runs by hand. + - name: join-credential + secret: + secretName: big-ops + defaultMode: 0440 + optional: true + volumeClaimTemplates: + - metadata: + name: data + spec: + # One volume per pod, never one shared: one process holds one file, and the pager takes + # an exclusive lock. + accessModes: [ReadWriteOnce] + # No `storageClassName`, so the cluster's default is used. Name one here if the default + # is network storage and this should be local - a database's latency is its disk's. + resources: + requests: + storage: 10Gi diff --git a/deploy/k8s/30-proxy.yaml b/deploy/k8s/30-proxy.yaml new file mode 100644 index 0000000..1879486 --- /dev/null +++ b/deploy/k8s/30-proxy.yaml @@ -0,0 +1,166 @@ +# The front door, and the only Service a client is given. +# +# It adds no routing - every node already plans a query and fans it out - so what it answers is +# "which node is alive", and that is the answer Kubernetes cannot produce here. `/ready` on a +# node is always `200`, with `serving` as a field in the body; an `httpGet` probe reads the +# status line and nothing else. This process reads the body, every couple of seconds, and takes +# a node that says `"serving":false` out of rotation without restarting anything. See +# crates/big-proxy/src/health.rs, which is written around exactly those two facts. +# +# It also adds a route allowlist: `/internal/*` is unreachable through it and cannot be made +# reachable, because it presents no client certificate either. +apiVersion: v1 +kind: Service +metadata: + name: big + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy +spec: + # **ClusterIP, and deliberately nothing else.** The compose file publishes to `127.0.0.1` + # because the hop below is in the clear; a ClusterIP is the same promise with the same + # boundary drawn around the cluster instead of the machine, and `kubectl port-forward` is the + # loopback hop. There is no Ingress and no LoadBalancer in this directory on purpose: adding + # one publishes a plaintext database port, and the fix is not an annotation but `--tls-cert` + # and `--tls-key` on this deployment and dropping `--insecure-no-tls` below. + type: ClusterIP + selector: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy + ports: + # 7654 outside, 7650 inside: a client that moves from the compose deployment to this one + # changes the host and nothing else. + - name: big + port: 7654 + targetPort: proxy +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: big-proxy + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy +spec: + # **Two, where compose has one.** A Deployment and not a StatefulSet because this is the one + # component here that genuinely is fungible: it opens no file, holds no lock, stores no + # credential and has no identity to keep. The compose file runs one of it because compose runs + # on one machine; a front door that is a single point of failure is the thing the front door + # was added to stop being. + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy + template: + metadata: + labels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy + spec: + terminationGracePeriodSeconds: 2 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: bigproxy + # A second image, and the separation is the point: this one has no engine, no SQL + # planner and no argon2 in it. See deploy/proxy/Dockerfile. + image: bigproxy:latest + imagePullPolicy: IfNotPresent + args: + - 0.0.0.0:7650 + - --cluster + - /etc/bigproxy/cluster/upstreams.toml + # Trusting the CA is not being trusted by it: this verifies the nodes' certificates + # and presents none of its own, which is why `/internal/*` stays out of reach. + - --upstream-ca + - /run/bigproxy/peer-ca.pem + # The hop from a client to here is in the clear, and the ClusterIP above is what + # bounds it. Give this `--tls-cert` and `--tls-key` and drop this flag before + # anything outside the cluster reaches it. + - --insecure-no-tls + ports: + - name: proxy + containerPort: 7650 + # Liveness: is this process answering. A proxy whose upstreams are all down is a + # healthy proxy - it is answering `503`, which is the correct answer - and restarting + # it would not bring a database back. + livenessProbe: + httpGet: { path: /health, port: proxy } + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + # Readiness: is there anywhere to send a request. `503` when nothing is in rotation, + # so this pod leaves the Service rather than accepting a request it can only refuse. + # It costs nothing to ask: the answer comes out of what the poller already knows, and + # the probe never touches a node. + readinessProbe: + httpGet: { path: /ready, port: proxy } + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 2 + resources: + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + readOnlyRootFilesystem: true + volumeMounts: + - name: run + mountPath: /run/bigproxy + - name: cluster + mountPath: /etc/bigproxy/cluster + readOnly: true + - name: secrets + mountPath: /etc/bigproxy/secrets + readOnly: true + volumes: + - name: run + emptyDir: {} + # **Not the seed the nodes read**, and the difference is the point: this list names + # every address a node could ever have, including pods that do not exist yet. The proxy + # takes `name` and `addr` and ignores the rest - it does not route by key - so a name it + # cannot reach is simply out of rotation, and one that appears is in rotation within a + # health interval. That is what makes `kubectl scale` reach a client with no restart + # here. The seed cannot be written that way: a node counts a majority out of it. + - name: cluster + configMap: + name: big-seed + items: + - key: upstreams.toml + path: upstreams.toml + # **The CA and nothing else.** Not a users file: this process authenticates nobody, it + # forwards the credential the client sent to a node that does. Not a node key: a + # certificate signed by that CA would name a node and would reach `/internal/*`, and + # there is deliberately no flag here that loads one. + - name: secrets + secret: + secretName: big-proxy + defaultMode: 0440 +--- +# **The one budget worth writing, and the nodes deliberately have none.** Draining a machine may +# take one of these two and must not take both - that is a real constraint an eviction can +# respect, because a second proxy is as good as the first. +# +# A node's is not that shape. Each range lives on exactly one node here, so evicting either one +# takes a range offline no matter what a budget says; `maxUnavailable: 0` would not protect the +# data, it would hang `kubectl drain` forever and call it protection. What makes a node evictable +# is a copy of its range, and that needs the third node docs/clustering.md describes. +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: big-proxy +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: big + app.kubernetes.io/component: proxy diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml new file mode 100644 index 0000000..0843cc6 --- /dev/null +++ b/deploy/k8s/kustomization.yaml @@ -0,0 +1,19 @@ +# `kubectl apply -k deploy/k8s` - after `./secrets.sh`, which is not listed here. +# +# **The Secrets are made by a script and not generated by kustomize**, on purpose. A +# `secretGenerator` would name them `big-node-a-` and roll the pods on every change, which +# for a credential file is usually the right instinct and here is not: rotating a node's +# certificate by restarting every node at once takes both ranges down, and the users file is +# read at startup anyway, so a rollout is a decision to make deliberately rather than a side +# effect of `apply`. It also keeps a private key out of every rendered manifest anybody pipes to +# a file. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: big + +resources: + - 00-namespace.yaml + - 10-config.yaml + - 20-nodes.yaml + - 30-proxy.yaml diff --git a/deploy/k8s/readme.md b/deploy/k8s/readme.md new file mode 100644 index 0000000..1ba6110 --- /dev/null +++ b/deploy/k8s/readme.md @@ -0,0 +1,277 @@ +# k8s + +Three nodes and a front door, and **`kubectl scale` is how the cluster grows**. The same binary +and the same flags as [`../cluster/`](../cluster/); what is different is that the shape is meant +to change here, so two flags that are off in the compose demo are on, and a pod that appears +gets itself into the cluster instead of waiting for somebody to write a file about it. + +## Running it + +There is no published image, so build both and put them where the cluster can see them: + +```sh +# From the repository root. +docker build -t big:latest . +docker build -t bigproxy:latest -f deploy/proxy/Dockerfile . + +kind load docker-image big:latest bigproxy:latest # or: minikube image load, or push and + # edit `image:` in the two manifests +``` + +Then the credentials and the manifests: + +```sh +cd deploy/k8s +./secrets.sh # a CA of its own, a certificate per pod name, and three Secrets. + # Prints three passwords - keep them. +kubectl apply -k . + +kubectl -n big rollout status statefulset/big +kubectl -n big port-forward svc/big 7654:7654 & +curl localhost:7654/ready +``` + +Two roles have to be made before the cluster is finished, and one of them is what lets it grow: + +```sh +ctl() { kubectl -n big exec -it big-0 -- \ + bigctl --addr https://big-0.nodes:7654 --ca-file /run/big/peer-ca.pem --user ops "$@"; } + +ctl sql "CREATE ROLE joiner" +ctl sql "GRANT OPERATE ON *.* TO joiner" # a new pod registers itself with this +ctl sql "CREATE ROLE dashboard" +ctl sql "GRANT OPERATE ON *.* TO dashboard" # /metrics, /verify +``` + +`bigctl` is in the image, the address is the node's *name* because that is what its certificate +says, and `--ca-file` is the CA the entrypoint staged inside the container. `--user` asks for the +password on the terminal, which is why that `exec` has `-it`; a script wants +`BIG_CREDENTIALS=` instead, one `user:password` line at mode 600, because a password given +as an argument is visible in `ps`. + +## Scaling out + +```sh +kubectl scale statefulset/big -n big --replicas=5 +ctl cluster topology +``` + +That is the whole of it, and what happens is worth following, because **only two of the four +steps are Kubernetes's**: + +1. **A certificate.** Already issued. `secrets.sh` mints one per *name* up to a ceiling, and a + name is cheap: a certificate grants nothing until the roster accepts it, and the roster + follows the agreement. So there is no `certs.sh` run and nothing to restart. +2. **A cluster file that names the cluster.** Written by the `join` initContainer, per pod, at + startup. A node's listener accepts a peer whose certificate names somebody on its roster, and + the roster it *starts* with comes from its own file - a node whose file named only itself + would refuse every peer that dialled it, have none of its own to dial, and sit there healthy + and alone. Once it is in touch the roster follows the agreement. +3. **A membership entry.** The same initContainer calls `cluster add-node`, before the daemon + starts. That registers the pod as a **learner**: it replicates the log, holds no range and + does not vote, so it neither answers for data it has not got nor raises the bar for an + election it could not help decide. +4. **A range.** Nobody asks for this. The balancer on the agreement's leader admits the learner + once it has caught up, and then cuts the tail of the shard space over to a node that holds + nothing - a split that moves no bytes. One decision per pass, each one committed. That is + `--balance`, and it is the flag `deploy/cluster` leaves off because two nodes and two ranges + are the thing that file is explaining. + +**The order in step 3 is not incidental.** A node that starts while the cluster has never heard +of it is a voter in a cluster of its own imagining, and it will stand for election once per +timeout - raising the term on every node that hears it - until somebody tells it otherwise. +Registered first, it is reached as a learner, and a learner does not stand. + +If the `joiner` role does not exist yet, or the `big-joiner` Secret is missing, the pod still +starts and its init container prints the one command to run: + +```sh +kubectl -n big logs big-4 -c join +# join: no credential mounted, so big-4 will not register itself. Run: +# join: bigctl cluster add-node big-4 big-4.nodes:7654 +``` + +**Past the ceiling.** `secrets.sh` issues eight names and the proxy is told about eight +addresses. `MAX=16 ./secrets.sh` issues the rest, and `upstreams.toml` in +[`10-config.yaml`](10-config.yaml) is the matching list - those two are the same list, and they +are the only two places a ceiling exists. + +## Scaling in is not the mirror image + +`kubectl scale --replicas=3` deletes `big-4`, and if `big-4` holds a range that range is +unanswerable until it comes back: nothing here is replicated, so the proxy picks a node that is +up and cannot pick data that is not there. **Take the work off it first:** + +```sh +ctl cluster drain big-4 # still votes, still coordinates; the balancer moves its ranges away +ctl cluster topology # wait until its `primary` column is empty +ctl cluster remove big-4 # refused while it still holds a range - that refusal is the point +kubectl scale statefulset/big -n big --replicas=4 +``` + +Draining is not removal, and removal is not deletion. Each is a separate decision because each +has a different failure: a node removed while it holds a range takes the only copy of it, and a +map that goes on naming a node nobody talks to is worse than one that says it is draining. + +The PVC is kept when a pod is scaled away (`whenScaled: Retain`), so scaling back up gives +`big-4` its file back rather than an empty one. That is what you want after an accidental scale +down and what you do not want after a real one - `kubectl delete pvc data-big-4` is the second +half of a scale-in that was meant. + +## The decisions + +### One StatefulSet, and `replicas` as the knob + +A node here is not interchangeable with the next one - it has a range, a certificate and a name - +which is the argument for one workload per node, and it is the wrong argument. What it produces +is a deployment where growing means writing YAML, and the cluster is already able to admit a +learner and give it work without being asked. So: one StatefulSet, pod names as node names, and +the parts that genuinely are per-node are handled where they belong - the certificate by a name +issued in advance, the cluster file by an initContainer, the range by the balancer. + +What ordinals cost is that the node names are Kubernetes's (`big-0`, not `a`), and that one +Secret is mounted by every pod. Both are written down where they bite: the second is on the +`secrets` volume in [`20-nodes.yaml`](20-nodes.yaml). + +### Three lists, three rules + +The mistake this deployment makes easy is treating them as one list. They are in one file, +[`10-config.yaml`](10-config.yaml), with the difference at the top: + +| | May name a node that does not exist | Why | +|---|---|---| +| `seed.toml` | **No** | A node with no log counts a majority out of it. Three entries with two pods up is a cluster that cannot elect anybody. | +| `upstreams.toml` | Yes | The proxy polls, finds it unreachable, leaves it out of rotation - and picks it up within a health interval when it appears. | +| the certificates | Yes | A certificate is checked against the roster, and the roster follows the agreement. An unused key grants nothing. | + +`seed.toml` names the founding three and is **not edited when the cluster grows**. Membership +after that is a committed decision, and `bigctl cluster topology` is where it is. A ConfigMap +edit does not reshape a running cluster and a rollout does not undo one. + +### Three nodes, not two + +Two is the size at which nothing works: a majority of two is two, so a two-node cluster can +neither fail a range over nor hand the schema-key namespace on - both are decisions a majority +has to commit at the moment one node is the one that went away. `deploy/cluster` runs two because +compose runs on one machine and the second node is there to show the shape. + +That is also why `--elect-schema-leader` is on here and off there: with two nodes it is inert +rather than merely cautious. + +### The peer Service publishes not-ready addresses + +A Service's endpoints are the ready ones by default. A node that has not passed its probe would +have no address; a peer that cannot reach it cannot help it settle the election that would make +it ready; both wait for each other on a cold start. `publishNotReadyAddresses: true` is the flag +that is not that deadlock, and it is also what a joining pod is dialled at before it is anything. + +### A probe can say "answering". It cannot say "serving" + +`GET /ready` on a node is **always `200`**. Whether it is actually serving its range is +`"serving": true|false` in the body, because a node that has lost touch with the agreement is +still perfectly healthy - it is one promotion away, and restarting it would help nothing. An +`httpGet` probe reads the status line and never the body. + +**That is what the proxy is for.** It polls `/ready` every couple of seconds, reads the field and +takes a node that says `false` out of rotation - see `crates/big-proxy/src/health.rs`, whose +module documentation is written around exactly this. So the Service a client is given is the +proxy's, and the proxy's own `/ready` *is* a real readiness signal: `503` when there is nowhere +to send a request, which takes that pod out of its Service. + +It also means `podManagementPolicy: OrderedReady` is safe here. If readiness meant "serving", an +ordered start would wait for a quorum that needs the next pod. + +### Why nothing runs as root, and the Secret is mode 0440 + +`big serve` refuses a users file or a private key that anyone but its owner can read. A Secret +volume cannot satisfy that directly: + +- its files are owned `root:`, and the daemon is uid 1000; +- `defaultMode: 0400` is therefore unreadable by the daemon - the entrypoint would silently stage + nothing, and the failure presents as `cannot read /run/big/users` rather than as a permission + error on the mount; +- `defaultMode: 0440` is readable, and is exactly what the check refuses. + +So the staging in [`../entrypoint.sh`](../entrypoint.sh) - written for bind mounts, whose +ownership the image cannot predict - is what makes a Secret work at all: it copies each file into +`/run/big` at mode 600, owned by whoever is running. `fsGroup: 1000` makes the source readable; +the private copy is what passes the check. The `join` initContainer does the same thing with its +credentials file, for the same reason. + +There is nothing left for root to do, so these pods start unprivileged: `runAsNonRoot`, no added +capabilities, `RuntimeDefault` seccomp, a read-only root filesystem with an `emptyDir` over +`/run/big`. The entrypoint notices it is not root and skips the privilege drop - the same path +`user:` in a compose file takes. The namespace enforces the `restricted` Pod Security profile, so +a manifest that regresses on any of that is rejected rather than quietly accepted. + +### ClusterIP, and no Ingress in this directory + +The compose file publishes to `127.0.0.1` because the hop from a client to the proxy is in the +clear. A ClusterIP is the same promise with the boundary drawn around the cluster instead of the +machine, and `kubectl port-forward` is the loopback hop. + +Adding an Ingress or a `type: LoadBalancer` here would publish a plaintext database port, so +neither is in this directory. The fix is not an annotation: give the proxy `--tls-cert` and +`--tls-key` from a Secret of its own and drop `--insecure-no-tls`, or terminate TLS in front of +it - [`../../runbook.md`](../../runbook.md) has a configuration that works. + +**`/verify`, `/repair` and `/admin/*` are not reachable through the proxy**, and that is the +allowlist rather than an omission: each asks about *one node*, and a proxy chooses which node +without telling you. `POST /admin/backup` is the sharpest case - through a front door it would +mean "back up a node, unspecified". `--allow-ops` turns them on if you want that anyway. + +## Rollouts, drains, and the range that has no copy + +No range here has a copy, so **stopping a node makes its range unanswerable until it comes +back**. Every operation below inherits that one fact. + +- **A rollout** deletes one pod at a time, so each node's range is down for as long as its pod + takes to come back. `kubectl rollout status` is the length of it. +- **`kubectl drain`** on a machine evicts the pod, which is the same thing. There is no + PodDisruptionBudget on the nodes on purpose: `maxUnavailable: 0` would not protect the data, it + would hang the drain forever and call that protection. What makes a node evictable is a copy of + its range. +- **The proxy has one**, `minAvailable: 1`, because a second proxy genuinely is as good as the + first - it holds no file, no lock and no identity. +- **A killed pod loses nothing.** A commit writes its pages, fsyncs, flips the meta page and + fsyncs again, so a process that dies leaves a file that is either before that flip or after it. + `terminationGracePeriodSeconds: 2` is generous. +- **The PVCs outlive the StatefulSet.** `kubectl delete -k .` leaves them behind and a re-apply + picks them back up: deleting a manifest must not be a way to delete a database. + +## Backups + +Per node, over the node's own port, and never with `big backup` - that subcommand wants the +exclusive lock the daemon is holding: + +```sh +kubectl -n big exec big-0 -- curl -sk -u ops:$PASSWORD -X POST \ + "https://127.0.0.1:7654/admin/backup?name=big-0-$(date +%F).db" +kubectl -n big cp big-0:/data/big-0-$(date +%F).db ./ +``` + +Each node holds its own range, so a copy of one node is a copy of one range, and the copies are +not one snapshot. Nothing here runs this for you: that is a CronJob and a bucket, and neither is +in this repository. + +## What is not here + +**An operator.** `bigctl cluster topology --format json` is the whole surface a placement +controller reads - which ranges exist, who holds each, what is moving - and `rebalance` is one +step against facts gathered afresh. What the initContainer does is the smallest thing that made +`kubectl scale` mean something: write a file, make one call. Everything past that - watching the +API, reacting to a node failure, deciding when to grow - is a controller's job and is not +pretended at here in eighty lines of shell. + +**A HorizontalPodAutoscaler.** Scaling out is cheap and scaling in is a sequence with a refusal +in the middle of it; a controller that scaled on CPU would eventually delete a pod holding the +only copy of a range. + +**Replication.** The balancer moves ranges; it does not make copies of them. A range with a copy +is what makes a rollout invisible and a node evictable, and setting one up is +[docs/clustering.md](../../docs/clustering.md). + +**Monitoring.** `/metrics` is Prometheus text on both, and the two are not the same route. The +proxy's is unauthenticated - it counts requests and latencies across a hop and says nothing about +anyone's data. A node's needs `OPERATE ON *.*`, is per node, and is not reachable through the +proxy, so a scrape config points at the `nodes` Service and carries the `dashboard` credential. diff --git a/deploy/k8s/secrets.sh b/deploy/k8s/secrets.sh new file mode 100755 index 0000000..216e2ea --- /dev/null +++ b/deploy/k8s/secrets.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# Everything this deployment needs on disk, and then in three Secrets. +# +# **A CA of its own, not `deploy/cluster`'s.** Two deployments sharing one CA is two deployments +# that can impersonate each other, and this one's node names are a StatefulSet's - `big-0`, +# `big-1` - rather than `a` and `b`. So it points the two scripts that already exist at a +# directory of its own; there is no cryptography here. +# +# ./secrets.sh # generate what is missing, then upload +# MAX=16 ./secrets.sh # room for sixteen pods rather than eight +# +# **Certificates are issued to the ceiling, not to the current size**, and that is safe in a way +# a membership list is not: a certificate names a node, and a node is let in by the *roster*, +# which follows the agreement rather than this directory. An unused key grants nothing. What it +# buys is that `kubectl scale big --replicas=6` needs no new certificate, no `certs.sh` run and +# no restart of anybody. +set -eu +cd "$(dirname "$0")" + +ns=${NS:-big} +out=${SECRETS:-secrets} +max=${MAX:-8} +big=${BIG:-big} + +command -v kubectl >/dev/null || { echo "no \`kubectl\` on PATH" >&2; exit 1; } + +# `big-0 big-1 ... big-`: the pod names a StatefulSet called `big` will hand out. +nodes="" +i=0 +while [ "$i" -lt "$max" ]; do + nodes="$nodes big-$i" + i=$((i + 1)) +done +nodes=${nodes# } + +# --------------------------------------------------------------------------------------------- +# On disk +# --------------------------------------------------------------------------------------------- + +if [ ! -f "$out/users" ]; then + command -v "$big" >/dev/null || { + echo "no \`$big\` on PATH; set BIG=/path/to/big. Hashing a password is argon2 and a" >&2 + echo "shell cannot do it - see ../cluster/users.sh." >&2 + exit 1 + } + SECRETS="../k8s/$out" ../cluster/users.sh + # **A third credential, and the reason it is not `ops`.** A pod that registers itself needs + # to reach `/admin/cluster/add-node`, which is `OPERATE ON *.*` - not superuser, and nothing + # else. Handing every pod the superuser password to save one `GRANT` would put full SQL + # access in a file every node can read. + # + # Named before the role exists, which is the right order and the same one `users.sh` uses + # for `dashboard`: a role is resolved by name on every request, so `CREATE ROLE joiner` + # after the cluster is up makes this credential work with no restart. Until then it + # authenticates and may do nothing - so joining does not work until an operator has decided + # it may. + joiner=$(od -An -tx1 -N24 /dev/urandom | tr -d ' \n') + printf '%s\n' "$joiner" | "$big" passwd "$out/users" set joiner --role joiner + printf 'joiner:%s\n' "$joiner" > "$out/join-credentials" + chmod 600 "$out/join-credentials" + echo + echo " joiner: $joiner (role \`joiner\`, which does not exist yet)" +fi + +if [ ! -f "$out/peer-ca.pem" ]; then + NODES="$nodes" SECRETS="../k8s/$out" ../cluster/certs.sh +else + # Already bootstrapped: issue only the names that are missing, against the CA that is there. + missing="" + for node in $nodes; do + [ -f "$out/$node.pem" ] || missing="$missing $node" + done + if [ -n "$missing" ]; then + # shellcheck disable=SC2086 + SECRETS="../k8s/$out" ../cluster/certs.sh $missing + fi +fi + +# --------------------------------------------------------------------------------------------- +# In the cluster +# --------------------------------------------------------------------------------------------- +# +# `create --dry-run=client | apply` rather than plain `create`: run twice, this rotates what is +# there instead of failing on the second run. A pod already running keeps what it staged at +# startup until it restarts - these files are read once. + +kubectl get namespace "$ns" >/dev/null 2>&1 || kubectl create namespace "$ns" + +# **One Secret for every pod**, because one StatefulSet is one PodSpec. Each pod stages only the +# certificate `--node` names, so no daemon presents anybody else's - but every pod could read +# every key. That is the trade one workload makes; 20-nodes.yaml says what to do if it is the +# wrong way round for you. +set -- --from-file=users="$out/users" --from-file=peer-ca.pem="$out/peer-ca.pem" +for node in $nodes; do + set -- "$@" --from-file="$node.pem=$out/$node.pem" --from-file="$node.key=$out/$node.key" +done +kubectl -n "$ns" create secret generic big-nodes "$@" \ + --dry-run=client -o yaml | kubectl -n "$ns" apply -f - + +# The CA alone. The proxy verifies node certificates and presents none - trusting a CA is not +# being trusted by it - and it authenticates nobody, so there is no users file here either. +kubectl -n "$ns" create secret generic big-proxy \ + --from-file=peer-ca.pem="$out/peer-ca.pem" \ + --dry-run=client -o yaml | kubectl -n "$ns" apply -f - + +# What a joining pod registers with. Optional in the manifest: without it a new pod still starts +# and prints the one `add-node` command an operator then runs by hand. +kubectl -n "$ns" create secret generic big-joiner \ + --from-file=credentials="$out/join-credentials" \ + --dry-run=client -o yaml | kubectl -n "$ns" apply -f - + +echo +echo "namespace $ns now holds:" +echo " big-nodes users, peer-ca.pem, and a certificate for each of: $nodes" +echo " big-proxy peer-ca.pem" +echo " big-joiner the credential a new pod registers itself with" +echo +echo "Once the cluster is up, make the two roles those credentials name:" +echo +echo " bigctl --user ops sql \"CREATE ROLE joiner\"" +echo " bigctl --user ops sql \"GRANT OPERATE ON *.* TO joiner\" # scaling out needs this" +echo " bigctl --user ops sql \"CREATE ROLE dashboard\"" +echo " bigctl --user ops sql \"GRANT OPERATE ON *.* TO dashboard\" # /metrics, /verify" +echo +echo "Until then \`kubectl scale\` starts a pod that cannot register itself, and says so in its" +echo "init container's log." diff --git a/deploy/readme.md b/deploy/readme.md index 3747fc3..bf735cb 100644 --- a/deploy/readme.md +++ b/deploy/readme.md @@ -1,12 +1,20 @@ # deploy -Two deployments, because they are two different things and pretending otherwise is how a +Three deployments, because they are three different things and pretending otherwise is how a single-node database ends up with a cluster's configuration and none of its guarantees. | | What it is | When | |---|---|---| | [`single/`](single/) | One node, every shard, no peers | Anything that fits on one machine | | [`cluster/`](cluster/) | Two ranges, one node each | More data than one machine holds | +| [`k8s/`](k8s/) | Three nodes that grow with `kubectl scale` | A cluster whose shape changes | + +**`cluster/` and `k8s/` are not the same deployment in two syntaxes.** The compose one is a fixed +shape being explained: two nodes, two ranges, and every flag that would redraw them turned off. +The Kubernetes one is a shape that changes: `--balance` is on, so a pod that appears is admitted +and given the tail of the shard space by the cluster's own leader, and a pod registers itself as +it starts. What that costs is a third node - a majority of two is two, so a two-node cluster can +commit neither a failover nor a schema-leader handover. **They are the same binary and the same code path.** `big serve` without `--cluster` builds itself a cluster of one and runs every request through the same coordinator a multi-machine deployment