diff --git a/crates/big-cluster/src/ddl.rs b/crates/big-cluster/src/ddl.rs index 27a63d1..1c111ef 100644 --- a/crates/big-cluster/src/ddl.rs +++ b/crates/big-cluster/src/ddl.rs @@ -108,6 +108,15 @@ impl Cluster

{ }) } + /// A mutex field that also declares which values it may hold. + /// + /// Travels the same leader-then-fan-out path every schema change takes, and it has to: a + /// node that missed the member list would accept a write the others refuse, which is the + /// kind of disagreement that shows up as a record existing on some nodes and not others. + pub fn create_enum(&self, table: &str, field: &str, members: Vec) -> Result { + self.ddl(&Ddl::CreateEnum { table: table.to_string(), field: field.to_string(), members }) + } + pub fn create_database(&self, name: &str) -> Result { self.ddl(&Ddl::CreateDatabase { name: name.to_string() }).map(|n| n == 1) } @@ -327,6 +336,9 @@ pub fn apply_ddl(api: &Api

, op: &Ddl) -> big_embed::Resul Ddl::CreateTimeQuantum { table, field, granularity } => { api.create_time_quantum(table, field, granularity.clone())? as u64 } + Ddl::CreateEnum { table, field, members } => { + api.create_enum(table, field, members.clone())? as u64 + } Ddl::DropTable { table } => api.drop_table(table)? as u64, // A table that is not here counts as nothing truncated rather than as a failure: the // leader has already ruled on whether it exists, so a peer without it is a peer that diff --git a/crates/big-cluster/src/query.rs b/crates/big-cluster/src/query.rs index a5b107b..f312e83 100644 --- a/crates/big-cluster/src/query.rs +++ b/crates/big-cluster/src/query.rs @@ -327,6 +327,11 @@ impl Cluster

{ big_embed::FieldKind::TimeQuantum => { self.create_time_quantum(table, &column.name, Vec::new()) } + // Decided on the declaration rather than on the kind, because an enum and a plain + // mutex are one `FieldKind` and only the member list tells them apart. + _ if !column.members.is_empty() => { + self.create_enum(table, &column.name, column.members.clone()) + } _ => self.create_field(table, &column.name, kind, column.bit_depth), } } @@ -936,6 +941,20 @@ impl Cluster

{ Some(select) => self.read_source(select, &name, opts)?, }; + // **`OVERWRITE` empties here, and the position is the whole of what it promises.** + // After the source query has been read whole and merged, so a `SELECT` that turns out + // to be unreadable leaves the table alone; before a single fact is written, so what + // lands is these rows and nothing older. It is still two operations with no transaction + // around them - a crash in between leaves the table empty, which `Insert::overwrite` + // says out loud rather than hiding. The atomic replacement is `EXCHANGE TABLES`. + // + // Row ids are allocated *after* this, because a truncate keeps the row keys but frees + // the fragments: allocating first would hand out ids against a record count that is + // about to become zero. + if insert.overwrite { + self.sql_truncate_table(&name, false)?; + } + // Asked for once for the whole statement rather than once per row: a round trip per // row would make a thousand-row insert a thousand round trips to one node. let allocated = match insert.id_at { diff --git a/crates/big-cluster/src/wire/fact.rs b/crates/big-cluster/src/wire/fact.rs index f266e66..0e0a5de 100644 --- a/crates/big-cluster/src/wire/fact.rs +++ b/crates/big-cluster/src/wire/fact.rs @@ -210,6 +210,16 @@ pub enum Ddl { field: String, granularity: Vec, }, + /// A `MUTEX` field that also declares which values it may hold. + /// + /// Its own variant rather than a `CreateField` with a list bolted on, for the reason + /// [`Ddl::CreateDecimal`] is one: the extra thing it carries is not optional, and a shape + /// where it might be absent is a shape where a peer can create the field without it. + CreateEnum { + table: String, + field: String, + members: Vec, + }, DropTable { table: String, }, @@ -346,6 +356,15 @@ impl Ddl { put_u8(&mut out, 13); put_str(&mut out, table); } + Self::CreateEnum { table, field, members } => { + put_u8(&mut out, 17); + put_str(&mut out, table); + put_str(&mut out, field); + put_count(&mut out, members.len()); + for m in members { + put_str(&mut out, m); + } + } Self::RenameTable { table, to } => { put_u8(&mut out, 15); put_str(&mut out, table); @@ -452,6 +471,18 @@ impl Ddl { 10 => Self::CreateRole { role: r.str()? }, 11 => Self::DropRole { role: r.str()? }, 13 => Self::TruncateTable { table: r.str()? }, + 17 => Self::CreateEnum { + table: r.str()?, + field: r.str()?, + members: { + let n = r.count()?; + let mut out = Vec::with_capacity(n.min(1024)); + for _ in 0..n { + out.push(r.str()?); + } + out + }, + }, 15 => Self::RenameTable { table: r.str()?, to: r.str()? }, 16 => Self::ExchangeTables { a: r.str()?, b: r.str()? }, 14 => Self::DropViewsBefore { diff --git a/crates/big-cluster/src/wire/plan.rs b/crates/big-cluster/src/wire/plan.rs index ec6c66b..cd641d5 100644 --- a/crates/big-cluster/src/wire/plan.rs +++ b/crates/big-cluster/src/wire/plan.rs @@ -63,6 +63,7 @@ mod rows_tag { pub const ALL: u8 = 9; pub const COMPARE_FLOAT: u8 = 10; pub const KEY_LIKE: u8 = 11; + pub const SAMPLE: u8 = 12; } pub fn put_rows(out: &mut Vec, rows: &Rows) { @@ -123,6 +124,11 @@ pub fn put_rows(out: &mut Vec, rows: &Rows) { put_rows(out, inner); } Rows::All => put_u8(out, rows_tag::ALL), + Rows::Sample { inner, stride } => { + put_u8(out, rows_tag::SAMPLE); + put_u32(out, *stride); + put_rows(out, inner); + } } } @@ -168,6 +174,12 @@ fn get_rows_at(r: &mut Reader<'_>, depth: usize) -> Result { ), rows_tag::NOT => Rows::Not(Box::new(get_rows_at(r, depth + 1)?)), rows_tag::ALL => Rows::All, + rows_tag::SAMPLE => { + // The stride before the inner set, so it is read before the recursion rather than + // after it - the same order it was written in. + let stride = r.u32()?; + Rows::Sample { inner: Box::new(get_rows_at(r, depth + 1)?), stride } + } tag => return Err(WireError::BadTag { what: "row set", tag }), }) } diff --git a/crates/big-cluster/src/wire/request.rs b/crates/big-cluster/src/wire/request.rs index 72999b6..59e8a87 100644 --- a/crates/big-cluster/src/wire/request.rs +++ b/crates/big-cluster/src/wire/request.rs @@ -637,6 +637,13 @@ pub fn put_schema( for g in &field.granularity { put_u8(out, g.as_char() as u8); } + // The declared enum members, which a repair needs for the same reason it needs the + // scale: without them the field comes back as the mutex it is stored as, and the + // check on the way in would be gone. + put_count(out, field.members.len()); + for m in &field.members { + put_str(out, m); + } } } put_count(out, views.len()); @@ -675,7 +682,19 @@ pub fn get_schema(bytes: &[u8]) -> Result<(Vec, Vec 5 +---- +count sum +3 24 + +# A stride past the record count is not an error, it is a small answer. +query +SELECT count(*) FROM smp SAMPLE 64 +---- +count +0 diff --git a/crates/big-cluster/tests/logic/schema.test b/crates/big-cluster/tests/logic/schema.test index b64d34b..81e3ea0 100644 --- a/crates/big-cluster/tests/logic/schema.test +++ b/crates/big-cluster/tests/logic/schema.test @@ -445,3 +445,60 @@ error EXCHANGE TABLES live AND typo ---- error: unknown_table + +## `ENUM`: the declaration is enforced, and it survives. +# +# Two claims, and neither is visible in translation. First that a value outside the list is +# refused on the way in — which is the whole of what declaring the members buys, because the +# mutex underneath would take any string. Second that `SHOW CREATE TABLE` says `ENUM(...)` and +# not `MUTEX`: the members live in the catalog beside the field, so the declared type is still +# there to print. + +statement +CREATE TABLE orders (status ENUM('new', 'paid', 'shipped'), amount UINT(32)) + +statement +INSERT INTO orders (_record_id, status, amount) VALUES (1, 'new', 10), (2, 'paid', 20) + +query +SELECT count(*) FROM orders WHERE status = 'paid' +---- +count +1 + +# A value the column was not declared to hold. The mutex underneath would have taken it. +error +INSERT INTO orders (_record_id, status, amount) VALUES (3, 'refunded', 30) +---- +error: malformed_line + +# And nothing was written: the statement failed before the first fact went out. +query +SELECT count(*) FROM orders +---- +count +2 + +# The declared type comes back, not the storage one. +query +SHOW CREATE TABLE orders +---- +statement +CREATE TABLE orders ( + status ENUM('new', 'paid', 'shipped'), + amount UINT(32) +) ENGINE = 'bitmap+columnar' + +# A plain mutex still takes anything, which is what says the check belongs to the declaration +# rather than to the kind. +statement +CREATE TABLE loose (status MUTEX) + +statement +INSERT INTO loose (_record_id, status) VALUES (1, 'anything at all') + +query +SELECT count(*) FROM loose WHERE status = 'anything at all' +---- +count +1 diff --git a/crates/big-db/src/catalog.rs b/crates/big-db/src/catalog.rs index b9fce33..f0da7e4 100644 --- a/crates/big-db/src/catalog.rs +++ b/crates/big-db/src/catalog.rs @@ -60,6 +60,7 @@ pub const KIND_SAVED_QUERY: u8 = kind::SAVED_QUERY; pub const KIND_SAVED_QUERY_TEXT: u8 = kind::SAVED_QUERY_TEXT; pub const KIND_ROLE: u8 = kind::ROLE; pub const KIND_GRANT: u8 = kind::GRANT; +pub const KIND_FIELD_ENUM: u8 = kind::FIELD_ENUM; const NAME_AT: usize = 24; @@ -254,6 +255,18 @@ pub struct FieldDef { /// Fixed scale for a decimal, so it can be stored as an integer. pub scale: i8, pub granularity: Vec, + /// The values an `ENUM` column was declared with, in the order written. Empty for every + /// other kind, and for a `MUTEX` that was declared as one. + /// + /// **Metadata, not storage.** The field is a `MUTEX` underneath and stores an interned row + /// id exactly as it would without this - which is the whole reason an enum costs no engine + /// change. What the list buys is the two things a `MUTEX` cannot say: a write outside it is + /// refused, and `SHOW CREATE TABLE` can name the type that was declared. + /// + /// The declared *numbers* of `Enum8('a' = 1)` are deliberately absent. They promise which + /// integer a value is stored as, and here the dictionary assigns that - so the parser + /// refuses the form rather than keeping a number nothing honours. + pub members: Vec, } /// A `SELECT` kept under a name, in the database that name is unique within. @@ -404,6 +417,27 @@ mod seq_of { pub const ROLE: u8 = 5; } +/// The separator between enum members on disk. +/// +/// NUL, because it is the one byte a member may not contain - checked where a column list is +/// read, so a member holding one never reaches this file. Any printable separator would be a +/// character somebody could legitimately put in a value. +const MEMBER_SEP: char = '\0'; + +/// The members as one string, for chunking into records. +fn join_members(members: &[String]) -> String { + members.join(&MEMBER_SEP.to_string()) +} + +/// The inverse. An empty string is no members rather than one empty member, which is what +/// distinguishes a field that declared none from one that declared `''`. +fn split_members(text: &str) -> Vec { + if text.is_empty() { + return Vec::new(); + } + text.split(MEMBER_SEP).map(str::to_string).collect() +} + /// Panics rather than truncating if a name got this far over-long: every public entry point /// checks first, so reaching here with one is a bug in this file and not bad input. fn put_name(b: &mut [u8], name: &str) { @@ -1146,8 +1180,24 @@ impl Catalog { b[12..16].copy_from_slice(&f.bit_depth.to_le_bytes()); b[16] = f.scale as u8; b[17] = gran_mask(&f.granularity); + // The total byte length of the member blob, in the header's free word - what lets + // the decoder tell a complete set of chunks from a torn one, exactly as a saved + // query's header carries its text length. Zero for every field that declares none, + // which is what an unwritten word already reads as. + let members = join_members(&f.members); + b[18..22].copy_from_slice(&(members.len() as u32).to_le_bytes()); put_name(&mut b, &f.name); out.push(b); + for (i, chunk) in members.as_bytes().chunks(TEXT_CHUNK_BYTES).enumerate() { + let mut b = vec![0u8; CATALOG_ENTRY_BYTES]; + b[0] = KIND_FIELD_ENUM; + b[2..4].copy_from_slice(&(chunk.len() as u16).to_le_bytes()); + b[4..8].copy_from_slice(&f.table.to_le_bytes()); + b[8..12].copy_from_slice(&f.id.to_le_bytes()); + b[12..16].copy_from_slice(&(i as u32).to_le_bytes()); + b[NAME_AT..NAME_AT + chunk.len()].copy_from_slice(chunk); + out.push(b); + } } for (k, m) in &self.fragments { let mut b = vec![0u8; CATALOG_ENTRY_BYTES]; @@ -1227,6 +1277,11 @@ impl Catalog { // were written in. let mut query_headers: BTreeMap = BTreeMap::new(); let mut query_text: BTreeMap<(QueryId, u32), Vec> = BTreeMap::new(); + // A field's declared enum members, staged the same way and for the same reason: the + // chunks and the header that says how many bytes they add up to arrive in whatever + // order the chain holds them. + let mut member_text: BTreeMap<(TableId, FieldId, u32), Vec> = BTreeMap::new(); + let mut member_len: BTreeMap<(TableId, FieldId), usize> = BTreeMap::new(); // Grants are staged for the same reason, one relation over: a grant names a role by id, // and nothing orders the chain so that the role record comes first. Collected here and @@ -1286,6 +1341,14 @@ impl Catalog { } query_text.insert((rd32(4), rd32(8)), e[NAME_AT..NAME_AT + n].to_vec()); } + KIND_FIELD_ENUM => { + let n = u16::from_le_bytes(e[2..4].try_into().unwrap()) as usize; + if n > TEXT_CHUNK_BYTES { + continue; + } + member_text + .insert((rd32(4), rd32(8), rd32(12)), e[NAME_AT..NAME_AT + n].to_vec()); + } KIND_FIELD => { let Some(name) = get_name(e) else { continue }; let Some(kind) = FieldKind::from_u8(e[1]) else { @@ -1303,7 +1366,13 @@ impl Catalog { bit_depth: rd32(12), scale: e[16] as i8, granularity: gran_from_mask(e[17]), + // Filled in after the pass, when every chunk has been seen: the records + // arrive in whatever order the chain holds them, so a field's members + // cannot be assembled at the record that announces how many bytes they + // are. The length is kept here to check against. + members: Vec::new(), }; + member_len.insert((def.table, def.id), rd32(18) as usize); c.field_ids.entry(def.table).or_default().insert(name, def.id); c.fields.insert((def.table, def.id), def); } @@ -1386,6 +1455,33 @@ impl Catalog { c.saved_queries.insert(id, SavedQuery { id, database, name, text }); } + // The enum members, reassembled. A field whose chunks do not add up to the length its + // header announced keeps **no** members rather than a truncated list, and that is + // fail-closed in the direction that matters: an empty list means "not an enum", so the + // column reads back as the `MUTEX` it is stored as and every value in it is still + // readable. A half-decoded list would refuse writes that were always legal. + for ((table, field), len) in member_len { + if len == 0 { + continue; + } + let mut bytes = Vec::with_capacity(len); + for ((_, _, _), chunk) in + member_text.range((table, field, 0)..=(table, field, u32::MAX)) + { + bytes.extend_from_slice(chunk); + } + if bytes.len() != len { + continue; + } + // Joined first, validated once, for the reason a saved query's text is: a chunk + // boundary falls mid-character often enough that per-record validation would reject + // ordinary names. + let Ok(text) = String::from_utf8(bytes) else { continue }; + if let Some(def) = c.fields.get_mut(&(table, field)) { + def.members = split_members(&text); + } + } + // The grants, now that every role record has gone past. One whose role is not there is // dropped: a privilege attached to nobody can never be exercised, and keeping it would // let a later `CREATE ROLE` that happened to reuse the id inherit it - which the diff --git a/crates/big-db/src/db/schema.rs b/crates/big-db/src/db/schema.rs index 63e09e4..50a60de 100644 --- a/crates/big-db/src/db/schema.rs +++ b/crates/big-db/src/db/schema.rs @@ -21,6 +21,27 @@ use super::*; +/// Everything about a field except which table it is in and what it is called. +/// +/// A struct rather than five more parameters, because five is where a reader stops being able to +/// tell which positional argument is which - and because every kind leaves most of them at their +/// default. [`Declared::of`] is that default, so a caller names only the part its kind actually +/// carries. +struct Declared { + kind: FieldKind, + bit_depth: u32, + scale: i8, + granularity: Vec, + members: Vec, +} + +impl Declared { + /// A field of this kind carrying nothing else: no depth, no scale, no views, no members. + fn of(kind: FieldKind) -> Self { + Self { kind, bit_depth: 0, scale: 0, granularity: Vec::new(), members: Vec::new() } + } +} + impl Db

{ /// Runs one transaction against the catalog, committing only if the closure succeeds. /// @@ -169,7 +190,16 @@ impl Db

{ scale: i8, ) -> Result { let table = table.into(); - self.declare(table, name, FieldKind::Decimal, bit_depth, scale, Vec::new()) + self.declare( + table, + name, + Declared { + kind: FieldKind::Decimal, + bit_depth, + scale, + ..Declared::of(FieldKind::Decimal) + }, + ) } /// A signed integer field. @@ -187,7 +217,7 @@ impl Db

{ bit_depth: u32, ) -> Result { let table = table.into(); - self.declare(table, name, FieldKind::SignedInt, bit_depth, 0, Vec::new()) + self.declare(table, name, Declared { bit_depth, ..Declared::of(FieldKind::SignedInt) }) } /// A time quantum field: a keyed field that also writes into one view per granularity. @@ -198,7 +228,7 @@ impl Db

{ granularity: Vec, ) -> Result { let table = table.into(); - self.declare(table, name, FieldKind::TimeQuantum, 0, 0, granularity) + self.declare(table, name, Declared { granularity, ..Declared::of(FieldKind::TimeQuantum) }) } pub fn create_field<'a>( @@ -209,17 +239,31 @@ impl Db

{ bit_depth: u32, ) -> Result { let table = table.into(); - self.declare(table, name, kind, bit_depth, 0, Vec::new()) + self.declare(table, name, Declared { bit_depth, ..Declared::of(kind) }) + } + + /// A `MUTEX` field that also declares which values it may hold - what SQL spells `ENUM`. + /// + /// **The kind is `MUTEX` and the members are metadata beside it**, which is the whole reason + /// an enum costs no engine change: one value per record over an interned dictionary is what + /// a mutex already is, and what a declared list adds is a check on the way in and a name on + /// the way out. A field with an empty list is a plain mutex, which is what every field + /// written before this existed reads back as. + pub fn create_enum<'a>( + &self, + table: impl Into>, + name: &str, + members: Vec, + ) -> Result { + let table = table.into(); + self.declare(table, name, Declared { members, ..Declared::of(FieldKind::Mutex) }) } fn declare<'a>( &self, table: impl Into>, name: &str, - kind: FieldKind, - bit_depth: u32, - scale: i8, - granularity: Vec, + d: Declared, ) -> Result { let table = table.into(); let mut w = self.write(); @@ -228,10 +272,11 @@ impl Db

{ id: 0, table: table_id, name: name.to_string(), - kind, - bit_depth, - scale, - granularity, + kind: d.kind, + bit_depth: d.bit_depth, + scale: d.scale, + granularity: d.granularity, + members: d.members, })?; w.commit()?; Ok(id) diff --git a/crates/big-db/tests/db.rs b/crates/big-db/tests/db.rs index 734b3ff..04aaa9c 100644 --- a/crates/big-db/tests/db.rs +++ b/crates/big-db/tests/db.rs @@ -1440,3 +1440,80 @@ fn exchanging_against_a_missing_table_moves_neither() { assert_eq!(d.catalog().lookup("a").unwrap().id, before); assert!(d.catalog().lookup("typo").is_none()); } + +/// **A declared enum member list survives a reopen, and is metadata beside a `MUTEX`.** +/// +/// The two halves worth pinning. First that the list comes back at all: it does not fit in a +/// catalog record, so it is chunked under its own record kind and reassembled on load, and a +/// chunk lost off the end has to be detectable — which is what the length in the field's header +/// is for. Second that the field is a plain mutex underneath, so an enum costs no engine change +/// and a reader that predates the record kind sees a working column rather than a broken one. +#[cfg(unix)] +#[test] +fn enum_members_survive_a_reopen_and_the_field_is_still_a_mutex() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("e.big"); + + // Long enough to span several 104-byte chunks, so the reassembly is actually exercised + // rather than fitting in one record by accident. + let members: Vec = (0..40).map(|i| format!("member_number_{i:03}")).collect(); + + { + let d = Db::open_path(&path).unwrap(); + d.create_table("t").unwrap(); + d.create_enum("t", "status", members.clone()).unwrap(); + let mut w = d.write(); + w.set_key("t", "status", 1, "member_number_007").unwrap(); + w.commit().unwrap(); + } + + let d = Db::open_path(&path).unwrap(); + let catalog = d.catalog(); + let def = catalog.field(0, "status").expect("the field is there"); + assert_eq!(def.members, members, "every member came back, in order"); + // Stored as a mutex, which is what makes this free below the catalog. + assert_eq!(def.kind, FieldKind::Mutex); + assert_eq!(d.read().by_key("t", "status", "member_number_007").unwrap(), vec![1]); +} + +/// A field that declares no members reads back with none — not one empty one. +#[test] +fn a_field_with_no_members_is_not_a_field_with_one_empty_member() { + let d = Db::in_memory().unwrap(); + d.create_table("t").unwrap(); + d.create_field("t", "plain", FieldKind::Mutex, 0).unwrap(); + assert!(d.catalog().field(0, "plain").unwrap().members.is_empty()); +} + +/// **A file written before enums existed still opens, and its mutex fields still work.** +/// +/// The backward half of the format change, and the one that cannot be checked by writing and +/// reading back with the same code. A `FIELD` record written by an older build carries zero in +/// the word that now holds the member-blob length, and no `FIELD_ENUM` records follow it — which +/// is exactly the shape a field declaring no members has today. So the test is that a field +/// whose header says zero reads back as a working mutex rather than as a broken enum. +#[cfg(unix)] +#[test] +fn a_field_written_without_members_reads_back_as_a_plain_mutex() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("old.big"); + + { + let d = Db::open_path(&path).unwrap(); + d.create_table("t").unwrap(); + // The call an older build had: no members, so the header word is zero and no member + // records are written. + d.create_field("t", "status", FieldKind::Mutex, 0).unwrap(); + let mut w = d.write(); + w.set_key("t", "status", 1, "anything at all").unwrap(); + w.commit().unwrap(); + } + + let d = Db::open_path(&path).unwrap(); + let catalog = d.catalog(); + let def = catalog.field(0, "status").expect("the field is there"); + assert!(def.members.is_empty(), "no members declared means no members read back"); + assert_eq!(def.kind, FieldKind::Mutex); + // And it still takes any value, which is what a mutex has always done. + assert_eq!(d.read().by_key("t", "status", "anything at all").unwrap(), vec![1]); +} diff --git a/crates/big-embed/src/fact.rs b/crates/big-embed/src/fact.rs index 6ee6bea..e93cbda 100644 --- a/crates/big-embed/src/fact.rs +++ b/crates/big-embed/src/fact.rs @@ -35,7 +35,10 @@ use big_sql::SqlError; /// **No `String` in it.** The happy path of an import runs once per fact over bodies of /// millions, so the error path is the only one that may allocate: this is the fact that /// something was wrong, and [`ValueError::why`] is the sentence, built on the way out. -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +// **Not , because one variant carries the members it refused against.** Naming the legal +// values is most of what that error is worth - a closed list is short, and "which values, then" +// is the only question it raises - so the list is carried and the trait is not. +#[derive(Clone, PartialEq, Eq, Debug)] pub enum ValueError { /// An unsigned integer field, given something that is not a whole number. NeedsNumber, @@ -47,6 +50,16 @@ pub enum ValueError { NeedsSeconds, /// A keyed field, given something that is not a string. NeedsKey, + /// An enum column, given a value it was not declared to hold. + /// + /// **The whole of what declaring the members buys**, and the reason it is a refusal rather + /// than a stored value: a mutex takes any string, so without this the declaration would be + /// a comment. Carries the members so the message can name them - a closed list is short by + /// construction, and "which values, then" is the only question this error raises. + NotAMember { + /// The values the column was declared with, in the order written. + members: Vec, + }, /// A float field, given something that is not a number. NeedsFloat, /// A float field, given a number it cannot hold: a NaN, or a magnitude past the range of a @@ -87,6 +100,10 @@ impl ValueError { format!("`{field}` takes `key@seconds`, and `{value}` is not a number") } Self::NeedsKey => format!("`{field}` needs a key, got `{value}`"), + Self::NotAMember { members } => { + let list = members.iter().map(|m| format!("'{m}'")).collect::>().join(", "); + format!("`{field}` was declared to hold one of {list}, got `{value}`") + } Self::NeedsFloat => format!("`{field}` needs a number, got `{value}`"), Self::FloatOutOfRange => { format!("`{field}` cannot hold `{value}`") @@ -178,10 +195,28 @@ pub fn from_text<'a>( Err(_) => return Err(ValueError::NeedsSeconds), }, }, - _ => Fact::Key { field, record, value }, + _ => { + // The other funnel a written value comes through - the import route's `field record + // value` lines - so the declaration is enforced on both or on neither. + member_check(info, value)?; + Fact::Key { field, record, value } + } }) } +/// Refuses a value an enum column was not declared to hold. +/// +/// A field with no members is not an enum, so this is the whole of the cost for every other +/// column: one `is_empty` on a `Vec` that is empty. The scan is linear because the list is +/// short by construction - a closed set somebody typed into a column definition - and building +/// an index per write would cost more than it saved. +fn member_check(info: &FieldInfo, value: &str) -> Result<(), ValueError> { + if info.members.is_empty() || info.members.iter().any(|m| m == value) { + return Ok(()); + } + Err(ValueError::NotAMember { members: info.members.clone() }) +} + /// One fact, from a value written as a SQL literal. /// /// The same table as [`from_text`], against the types a parser already told apart. The one @@ -252,7 +287,13 @@ pub fn from_literal<'a>( (FieldKind::TimeQuantum, Literal::Str(s)) => from_text(field, info, record, s)?, (FieldKind::TimeQuantum, _) => return Err(ValueError::NeedsKey), - (_, Literal::Str(s)) => Fact::Key { field, record, value: s.as_str() }, + (_, Literal::Str(s)) => { + // Checked here rather than at the storage layer, which sees a mutex and has no list + // to check against: the declaration lives in the catalog beside the field, and this + // is the one funnel every written value passes through. + member_check(info, s)?; + Fact::Key { field, record, value: s.as_str() } + } (_, _) => return Err(ValueError::NeedsKey), }) } diff --git a/crates/big-embed/src/introspect.rs b/crates/big-embed/src/introspect.rs index fdf4a57..0d7f1c1 100644 --- a/crates/big-embed/src/introspect.rs +++ b/crates/big-embed/src/introspect.rs @@ -36,6 +36,10 @@ use big_sql::{Column as SqlColumn, ColumnKind as SqlColumnKind}; /// The field kind a column list's type means. pub fn kind_of(kind: SqlColumnKind) -> FieldKind { match kind { + // Stored as the mutex it is. The member list travels beside the kind rather than inside + // it - see `big_sql::Column::members` - which is what keeps an enum free of any engine + // change at all. + SqlColumnKind::Enum => FieldKind::Mutex, SqlColumnKind::Set => FieldKind::Set, SqlColumnKind::Mutex => FieldKind::Mutex, SqlColumnKind::Bool => FieldKind::Bool, @@ -279,9 +283,16 @@ pub fn show_create( .iter() .map(|f| SqlColumn { name: f.name.clone(), - kind: sql_kind_of(f.kind), + // **The declared type, which the storage kind alone cannot say.** A mutex and an + // enum are one `FieldKind`; what tells them apart is whether a member list was + // declared, so that is what decides here rather than the kind. + kind: match f.members.is_empty() { + true => sql_kind_of(f.kind), + false => SqlColumnKind::Enum, + }, bit_depth: f.bit_depth, scale: (f.kind == FieldKind::Decimal).then_some(f.scale), + members: f.members.clone(), }) .collect(); // Qualified whenever the table is not in the default database, so the statement this diff --git a/crates/big-embed/src/lib.rs b/crates/big-embed/src/lib.rs index ce03f6f..b5958e5 100644 --- a/crates/big-embed/src/lib.rs +++ b/crates/big-embed/src/lib.rs @@ -699,6 +699,14 @@ impl Api

{ Ok(self.db.create_time_quantum(table, field, granularity)?) } + /// A `MUTEX` field that also declares which values it may hold - what SQL spells `ENUM`. + /// + /// See [`big_db::Db::create_enum`]: the kind stored is a mutex and the members are metadata + /// beside it, which is what keeps an enum free of any engine change. + pub fn create_enum(&self, table: &str, field: &str, members: Vec) -> Result { + Ok(self.db.create_enum(table, field, members)?) + } + /// A snapshot of the schema, owned outright. /// /// The catalog lives behind a lock; handing out a guard would let a caller hold it while diff --git a/crates/big-embed/src/schema.rs b/crates/big-embed/src/schema.rs index 40d0589..b14cbdf 100644 --- a/crates/big-embed/src/schema.rs +++ b/crates/big-embed/src/schema.rs @@ -55,6 +55,12 @@ pub struct FieldInfo { pub scale: i8, /// The views a time quantum field writes. Empty for every other kind. pub granularity: Vec, + /// The values an `ENUM` was declared with. Empty for every other kind, and for a `MUTEX` + /// declared as one. + /// + /// Reported for the reason a decimal's scale is: without it a client cannot tell an enum + /// from the mutex it is stored as, and could not recreate the field as it was declared. + pub members: Vec, } /// Copies the catalog into owned values, dropping the lock before returning. @@ -85,6 +91,7 @@ pub(crate) fn snapshot(catalog: &Catalog) -> Vec { .fields_of(table.id) .map(|f| FieldInfo { name: f.name.clone(), + members: f.members.clone(), kind: f.kind, bit_depth: f.bit_depth, scale: f.scale, diff --git a/crates/big-embed/tests/introspect.rs b/crates/big-embed/tests/introspect.rs index 7266c53..a2b55e3 100644 --- a/crates/big-embed/tests/introspect.rs +++ b/crates/big-embed/tests/introspect.rs @@ -24,7 +24,14 @@ use big_embed::{introspect, Datum, Fact, FieldInfo, FieldKind, TableEngine, Tabl use big_plan::Literal; fn field(name: &str, kind: FieldKind, bit_depth: u32, scale: i8) -> FieldInfo { - FieldInfo { name: name.to_string(), kind, bit_depth, scale, granularity: Vec::new() } + FieldInfo { + name: name.to_string(), + kind, + bit_depth, + scale, + granularity: Vec::new(), + members: Vec::new(), + } } fn schema() -> Vec { @@ -199,3 +206,32 @@ fn a_value_a_field_cannot_hold_says_what_the_field_wanted() { assert_eq!(big_embed::fact::written(&Literal::Dec { units: 5, scale: 3 }), "0.005"); assert_eq!(big_embed::fact::written(&Literal::Sint(-5)), "-5"); } + +/// **The refusal names the values the column was declared to hold.** +/// +/// The code a client branches on is the generic one for a value a field cannot take, and that is +/// right — what to do about it is the same. The *message* is where an enum earns its keep: a +/// closed list is short by construction, so naming it turns "that value is not allowed" into +/// "these are", which is the only question the error raises. +#[test] +fn a_value_outside_an_enum_is_refused_with_the_members_named() { + let info = big_embed::FieldInfo { + name: "status".to_string(), + kind: big_embed::FieldKind::Mutex, + bit_depth: 0, + scale: 0, + granularity: Vec::new(), + members: ["new", "paid"].map(str::to_string).to_vec(), + }; + + let err = big_embed::fact::from_text("status", &info, 1, "refunded") + .expect_err("a value outside the list is refused"); + let why = err.why("status", "refunded"); + assert!(why.contains("'new'") && why.contains("'paid'"), "the members are named: {why}"); + + // A declared value goes through, and so does anything at all on a field that declared none - + // which is what says the check belongs to the declaration rather than to the kind. + assert!(big_embed::fact::from_text("status", &info, 1, "paid").is_ok()); + let loose = big_embed::FieldInfo { members: Vec::new(), ..info }; + assert!(big_embed::fact::from_text("status", &loose, 1, "anything at all").is_ok()); +} diff --git a/crates/big-exec/src/lib.rs b/crates/big-exec/src/lib.rs index 1834001..e84eb3d 100644 --- a/crates/big-exec/src/lib.rs +++ b/crates/big-exec/src/lib.rs @@ -24,6 +24,8 @@ #![deny(unsafe_code)] +use std::collections::BTreeMap; + use big_db::catalog::{Catalog, FieldKind}; use big_db::{DbRead, Matches, RangeOp, RecordId, RowId}; use big_pager::Pager; @@ -768,9 +770,68 @@ fn eval(db: &DbRead<'_, P>, table: &str, rows: &Rows) -> Result // both would silently drop every part after the first empty one. Rows::Intersect(parts) => fold(db, table, parts, Matches::and, StopWhenEmpty::Yes)?, Rows::Union(parts) => fold(db, table, parts, Matches::or, StopWhenEmpty::No)?, + + // **One AND per container, and never a record decoded.** A container holds 65,536 + // consecutive record ids, so "every `stride`th id" is the same pattern in every + // container - built once here and intersected with each one the inner set touches. + // + // That is also why the stride is a power of two: the pattern only repeats when it + // divides the container, and a stride that did not would be right in one container and + // wrong in the next. `big_plan` refuses the rest at the call. + Rows::Sample { inner, stride } => sample(eval(db, table, inner)?, *stride), }) } +/// One record in every `stride`, by record id. +/// +/// **The unit is the id's low bits, not a block of ids**, and that is the whole of why this is +/// a sample rather than a slice. Ids are handed out in write order, so every eighth *container* +/// would be every eighth window of time - a subset that leans whichever way the data drifted. +/// Every eighth *id* is spread evenly through each of those windows. +/// +/// Costs one container-sized AND per container the set touches. Nothing is decoded, so a +/// sampled count is as cheap as the count it came from. +fn sample(matches: Matches, stride: u32) -> Matches { + // A container is addressed by a `u16` offset, so this is its width - written from the type + // rather than as a literal, because the two cannot then drift apart. + const WIDTH: u64 = u16::MAX as u64 + 1; + let stride = u64::from(stride.max(2)); + + // **Keyed by phase, not by container.** A container beginning at id `c * 65536` starts + // part-way through the repeating pattern unless the stride divides the width, and the + // offset into it is arithmetic. There are at most `stride` distinct phases and usually far + // fewer, so this caches what would otherwise be rebuilt per container - and the common case + // of a stride that does divide the width has exactly one entry. + let mut masks: BTreeMap = BTreeMap::new(); + + let mut out = Matches::new(); + // Collected first because the loop reads the same `matches` it is iterating the shards of. + let shards: Vec<_> = matches.shards().collect(); + for shard in shards { + let Some(rows) = matches.get(shard) else { continue }; + let mut per_shard = big_db::RowSet::new(); + for (slot, _) in rows.iter() { + // The id of this container's first record, modulo the stride: where in the pattern + // it begins. `slot` is the container's index within the shard, and a shard is + // `SHARD_WIDTH` records, so the two together give the record id. + let first = shard.wrapping_mul(big_db::SHARD_WIDTH).wrapping_add(slot * WIDTH); + let phase = first % stride; + // The first offset inside this container that is on the pattern. + let start = (stride - phase) % stride; + let mask = masks.entry(phase).or_insert_with(|| { + big_db::Container::from_values( + (start..WIDTH).step_by(stride as usize).map(|v| v as u16), + ) + }); + per_shard.insert(slot, mask.clone()); + } + // `RowSet::insert` drops an empty container, so a shard that samples to nothing + // contributes nothing - the same shape `Matches` already holds for an empty shard. + out.insert(shard, rows.and(&per_shard)); + } + out +} + /// Whether an empty accumulator can still be changed by what is left. #[derive(Clone, Copy, PartialEq, Eq)] enum StopWhenEmpty { diff --git a/crates/big-http/src/routes/query.rs b/crates/big-http/src/routes/query.rs index 1854036..c2cb162 100644 --- a/crates/big-http/src/routes/query.rs +++ b/crates/big-http/src/routes/query.rs @@ -773,7 +773,14 @@ mod parse_tests { use super::*; fn field(name: &str, kind: FieldKind) -> FieldInfo { - FieldInfo { name: name.to_string(), kind, bit_depth: 20, scale: 0, granularity: Vec::new() } + FieldInfo { + name: name.to_string(), + kind, + bit_depth: 20, + scale: 0, + granularity: Vec::new(), + members: Vec::new(), + } } /// The word-at-a-time scan against the obvious one, at every length and every offset a diff --git a/crates/big-page/src/record.rs b/crates/big-page/src/record.rs index f61c7ce..4bdd1a7 100644 --- a/crates/big-page/src/record.rs +++ b/crates/big-page/src/record.rs @@ -69,10 +69,22 @@ pub mod kind { /// are never reissued, so a table dropped and recreated under the same name gets a new id /// and cannot inherit the privileges the old one carried. pub const GRANT: u8 = 11; + /// One 104-byte slice of a field's declared enum members, under its table and field id. + /// + /// **The second payload in this format that spans records**, and it follows + /// [`SAVED_QUERY_TEXT`]'s shape for the same reason: a record is a fixed width and a list of + /// member names is not. The members are joined by a NUL byte, which is the one byte a + /// member may not contain, and cut wherever 104 bytes falls - including mid-character, so + /// the chunks are joined before they are validated. + /// + /// Purely additive. A reader that predates this kind skips the entry and sees the field as + /// the `MUTEX` it is stored as, which is what it was before enums existed - the same + /// backward story [`DATABASE`] and [`SEQ`] tell. + pub const FIELD_ENUM: u8 = 12; } /// Every kind, so adding one without checking it against the others is not possible. -pub const ALL_KINDS: [u8; 11] = [ +pub const ALL_KINDS: [u8; 12] = [ kind::TABLE, kind::FIELD, kind::VIEW, @@ -84,6 +96,7 @@ pub const ALL_KINDS: [u8; 11] = [ kind::SAVED_QUERY_TEXT, kind::ROLE, kind::GRANT, + kind::FIELD_ENUM, ]; // Distinctness, checked at compile time. Two crates allocate out of this space and cannot see diff --git a/crates/big-plan/src/explain.rs b/crates/big-plan/src/explain.rs index 3773c0d..7aa3456 100644 --- a/crates/big-plan/src/explain.rs +++ b/crates/big-plan/src/explain.rs @@ -150,6 +150,7 @@ fn unfolded_aggregate(plan: &Plan) -> Option<&Plan> { fn rows_head(rows: &Rows) -> String { match rows { Rows::All => "all".to_string(), + Rows::Sample { stride, .. } => format!("sample 1 in {stride}"), Rows::Compare { field, op, value } => format!("{field} {} {value}", cmp(*op)), // The marker is not decoration. A signed field's bound is an `i64` all the way down and // a positive one prints identically to an unsigned bound, so without it two different @@ -187,6 +188,7 @@ fn rows_head(rows: &Rows) -> String { fn rows_kids(rows: &Rows) -> Vec<&Rows> { match rows { Rows::Intersect(parts) | Rows::Union(parts) => parts.iter().collect(), + Rows::Sample { inner, .. } => vec![inner.as_ref()], Rows::Difference(a, b) => vec![a, b], Rows::Not(inner) => vec![inner], _ => Vec::new(), diff --git a/crates/big-plan/src/plan.rs b/crates/big-plan/src/plan.rs index 559215a..7fcfc9f 100644 --- a/crates/big-plan/src/plan.rs +++ b/crates/big-plan/src/plan.rs @@ -113,6 +113,23 @@ pub enum Rows { /// Complement against every record that exists, which is why the table is carried along. Not(Box), All, + /// One record in every `stride`, by record id. + /// + /// **Sampled on the *low bits of the id*, not on blocks of them**, and the difference is the + /// whole reason this is answerable at all. A roaring container holds 65,536 consecutive + /// ids, so "every eighth id" is a fixed bit pattern inside each container - one word + /// constant, ANDed - which costs a pass over containers and never decodes a record. Taking + /// every eighth *container* would be as cheap and would be a different thing: ids are + /// handed out in write order, so a block of them is a window of time, and a number drawn + /// from one would lean whichever way the data drifted. + /// + /// `stride` is at least two. It need not divide a container: the pattern repeats every + /// `stride` ids whatever it is, and a container that starts part-way through one begins at + /// a known phase - so the mask is still built rather than walked. + Sample { + inner: Box, + stride: u32, + }, } /// One column a grouping is over, and how its values become sets of records. @@ -824,6 +841,7 @@ impl Ctx<'_, S> { "Like" => self.like(call, false), "ILike" => self.like(call, true), "Not" => Ok(Rows::Not(Box::new(self.one_rows_arg("Not", &call.args)?))), + "Sample" => self.sample(call), "Intersect" => Ok(Rows::Intersect(self.rows_args("Intersect", &call.args)?)), "Union" => Ok(Rows::Union(self.rows_args("Union", &call.args)?)), "Difference" => match call.args.as_slice() { @@ -971,6 +989,36 @@ impl Ctx<'_, S> { crate::rounded::rows(field, self.class(field)?, by, op, value) } + /// `Sample(, stride=)`: one record in every `n`, by record id. + /// + /// `stride` is named rather than positional for the reason a field is on an aggregate: the + /// two arguments are not interchangeable and a reader should not have to remember the order. + /// + /// **Any stride from two upwards**, because the mask does not have to divide a container to + /// be a mask: the pattern repeats every `stride` ids, and a container beginning part-way + /// through one begins at a phase that is arithmetic rather than a search. Rounding a stride + /// to a nearby power of two would be a different sample wearing the right name. + fn sample(&self, call: &Call) -> Result { + const WANT: &str = "a bitmap and stride=<2 or more>"; + let (mut inner, mut stride) = (None, None); + for arg in &call.args { + match arg { + Expr::Call(c) => inner = Some(self.rows(c)?), + Expr::Named { name, value } if name == "stride" => match value.as_ref() { + Expr::Literal(Literal::Int(n)) if (2..=u64::from(u32::MAX)).contains(n) => { + stride = Some(*n as u32) + } + _ => return Err(PlanError::BadArgument { call: "Sample", want: WANT }), + }, + _ => return Err(PlanError::BadArgument { call: "Sample", want: WANT }), + } + } + let (Some(inner), Some(stride)) = (inner, stride) else { + return Err(PlanError::Arity { call: "Sample", want: WANT, got: call.args.len() }); + }; + Ok(Rows::Sample { inner: Box::new(inner), stride }) + } + fn row(&self, call: &Call) -> Result { // A time window is the only form with more than one argument, so it is recognised // before the shapes that insist on exactly one. diff --git a/crates/big-sql/readme.md b/crates/big-sql/readme.md index b046262..3b4b72b 100644 --- a/crates/big-sql/readme.md +++ b/crates/big-sql/readme.md @@ -250,6 +250,59 @@ The new name is bare and both tables are in one database. A rename that also mov changes wearing one name, and it is a syntax error rather than a refusal — there is nothing to write instead of it. +## A column with a closed set of values + +```sql +CREATE TABLE orders (status ENUM('new', 'paid', 'shipped')) +``` + +**Stored as a `MUTEX`, with the members as metadata beside it.** One value per record over an +interned dictionary is what a mutex already is, so an enum costs no new `FieldKind`, no engine +change and no storage format — the list travels in the catalog next to the field, in its own +record kind because it does not fit in one. + +What the list buys is the two things a mutex cannot say. A value outside it is refused on the way +in, naming the values that are allowed; and `SHOW CREATE TABLE` answers `ENUM('new', 'paid', +'shipped')` rather than `MUTEX`, because the declaration is still there to print. A field that +declares no members is a plain mutex and takes anything, which is what says the check belongs to +the declaration rather than to the kind — and is also what every field written before this +existed reads back as. + +`Enum8` and `Enum16` are the same declaration; the width names how many members fit and the +dictionary has no such ceiling. The `'a' = 1` form is refused (`sql_enum_type`): the number says +which integer the value is stored as, and here the dictionary assigns that, so keeping it would +be a promise nothing honours. A repeated member is refused too — the second is unreachable, and a +list holding fewer values than it names would make `SHOW CREATE TABLE` disagree with what was +typed. + +## Sampling, and replacing a table's contents + +```sql +SELECT count(*) FROM t SAMPLE 10 -- also `SAMPLE 1/10` and `SAMPLE 0.1` +INSERT OVERWRITE TABLE t (a) SELECT a FROM u +``` + +**A sample is on the record id's low bits, not on a block of them**, and that is what makes it a +sample. A container holds 65,536 consecutive ids, so "every tenth id" is a pattern inside each +one — built once per phase and intersected, costing a pass over containers and never decoding a +record. Every tenth *container* would be as cheap and would be a different thing: ids are handed +out in write order, so a block of them is a window of time and a number drawn from one would +lean whichever way the data drifted. + +The three spellings are one stride, and `0.1` is read through the decimal it parsed as — units +and a scale, never a float — so it is exactly one in ten. A fraction that is not one over a whole +number has no stride and is refused rather than rounded (`sql_no_sample`), because one in 3.33 +answered as one in three is a tenth more data than was asked for. It wraps the `WHERE` rather +than sitting beside it: a tenth of the records over five, not a tenth of every record of which +some are over five. + +`INSERT OVERWRITE` empties the table and then writes. It is two operations with nothing wrapping +them, and what it buys over writing `TRUNCATE` and `INSERT` separately is the order: the source +query is read whole and merged *before* anything is cleared, so a `SELECT` that turns out to be +unreadable leaves the table alone. A crash between the two still leaves it empty, which is said +rather than hidden — the atomic replacement is `EXCHANGE TABLES`. It demands `DELETE` as well as +`INSERT`, because destroying what was there is what it does. + ## Rows from nothing ```sql @@ -388,18 +441,14 @@ and `IPv6` are 128 bits where a bit-sliced value stops at 64, so they are `TEXT` and compared as bitmaps, but without a number's ordering. **The P2 tier of the OLAP surface**, each with the local spelling in its sentence rather than the -news that a feature is absent. `SAMPLE` (`sql_no_sample`): there is no rank-select over a -container, and the units that are cheap to skip whole are windows of write order rather than -random subsets. `INSERT OVERWRITE` (`sql_no_overwrite`): a truncate and an insert, which nothing -wraps in one transaction — so two statements, or `EXCHANGE TABLES` for an atomic replacement. -`COPY INTO` (`sql_no_copy_into`): loading is `POST /table/{t}/import`. `CREATE TEMPORARY TABLE` +news that a feature is absent. `COPY INTO` (`sql_no_copy_into`): loading is `POST /table/{t}/import`. `CREATE TEMPORARY TABLE` (`sql_no_temporary_table`): there is no session for one to be temporary to. `PARTITION BY` and `DISTRIBUTED BY` (`sql_no_distribution`): a shard is `record_id >> 20`, computed rather than declared. `MATERIALIZED` and `ALIAS` columns (`sql_computed_column`): one is a view, the other a materialised one. `Array`, `Map`, `Tuple` (`sql_composite_type`) and `ARRAY JOIN` (`sql_no_array_join`) and the `array*` functions (`sql_array_function`): a keyed column already holds many values per record, so `has(c, x)` is `c = 'x'` and one row per element is -`GROUP BY c`. `Enum8` (`sql_enum_type`): `MUTEX` is that storage already. `neighbor`, +`GROUP BY c`. `neighbor`, `windowFunnel` and `retention` (`sql_no_sequence_function`). And `s3`, `url`, `file` and `generateRandom` (`sql_no_table_function`), where `numbers(n)` is the one that works. diff --git a/crates/big-sql/src/ast.rs b/crates/big-sql/src/ast.rs index ab1cb6d..39ab50f 100644 --- a/crates/big-sql/src/ast.rs +++ b/crates/big-sql/src/ast.rs @@ -185,6 +185,12 @@ pub struct Select { /// Every one of them has to key its table on the same column the others do: what makes /// several joins answerable is that they are one star around one shared key. pub joins: Vec, + /// `SAMPLE`: one record in every `n`, absent when the statement asked for all of them. + /// + /// **Held as the stride rather than as the fraction that was written**, because the stride + /// is what the engine does: `SAMPLE 0.1` and `SAMPLE 1/10` are one number by the time + /// anything reads this, so nothing downstream has to know which was typed. + pub sample: Option, /// `WHERE`, absent when every record is in play. pub filter: Option, /// `GROUP BY`: the columns, and the boundary each is rounded to. diff --git a/crates/big-sql/src/ddl.rs b/crates/big-sql/src/ddl.rs index 5de81aa..491d9c9 100644 --- a/crates/big-sql/src/ddl.rs +++ b/crates/big-sql/src/ddl.rs @@ -323,6 +323,13 @@ pub struct Column { pub bit_depth: u32, /// Digits after the point, and `Some` exactly when the kind is [`ColumnKind::Decimal`]. pub scale: Option, + /// The values an `ENUM` may hold, in the order written. Empty for every other kind. + /// + /// **Metadata beside a `MUTEX`, not a kind of its own below the catalog.** One value per + /// record over an interned dictionary is what a mutex already is; what the list adds is a + /// value outside it being refused on the way in, and the declared type surviving + /// `SHOW CREATE TABLE`. + pub members: Vec, } /// What a field stores, as a column list may name it. @@ -338,6 +345,12 @@ pub struct Column { /// what `BIGINT` has always been. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum ColumnKind { + /// `ENUM('a', 'b')`: a `MUTEX` that also declares which values it may hold. + /// + /// A `ColumnKind` rather than a flag on [`Column`] so that the layers that match on a kind + /// have to say what they do with it - which is how `SHOW CREATE TABLE` came to print the + /// declared type rather than the storage one. + Enum, Set, Mutex, Bool, @@ -358,6 +371,10 @@ impl ColumnKind { /// written. pub fn as_str(self) -> &'static str { match self { + // **The storage spelling, which is a mutex.** This is the word the field route + // takes, and that route has no way to carry a member list - so a field created + // through it is the plain mutex an enum is stored as, which is exactly right. + Self::Enum => "mutex", Self::Set => "set", Self::Mutex => "mutex", Self::Bool => "bool", diff --git a/crates/big-sql/src/error.rs b/crates/big-sql/src/error.rs index ef77b59..8be1f4a 100644 --- a/crates/big-sql/src/error.rs +++ b/crates/big-sql/src/error.rs @@ -401,17 +401,20 @@ pub enum Refused { /// these are one family of questions, and what each needs is a different local spelling /// rather than a different kind of answer. SequenceFunction, - /// `SAMPLE 0.1` after a `FROM`, or `SAMPLE BY expr` in a `CREATE TABLE`. + /// A `SAMPLE` fraction that names no whole stride, or `SAMPLE BY` in a `CREATE TABLE`. /// - /// There is no rank-select over a roaring container, so there is no way to take every tenth - /// matching record without decoding the ones in between. What is cheap is a *container* or a - /// shard, and neither is a random subset: record ids are handed out in write order, so a - /// container is a window of time wearing the shape of a sample. + /// **The clause itself is answered** - one record in every `n`, masked on the id's low bits. + /// What is refused is a fraction that is not one over a whole number, because rounding it + /// would answer a different question under the name of this one, and `SAMPLE BY`, which + /// names the column a sample is drawn on: here the unit is the record id, which is not a + /// column and cannot be chosen. Sample, - /// `INSERT OVERWRITE`. + /// `INSERT OVERWRITE ... PARTITION (...)`. /// - /// It is a truncate and an insert, and nothing here wraps two of those in one transaction - - /// so a statement that looked atomic would not be, which is worse than two that do not. + /// **The statement itself is answered now**; what stays refused is the clause that aims it + /// at a partition, because a shard here is a function of the record id and there is nothing + /// for one to name. Narrowed rather than removed, which is the widening + /// `docs/versioning.md` allows. Overwrite, /// `COPY INTO t FROM 's3://...'`. CopyInto, @@ -439,11 +442,12 @@ pub enum Refused { /// column already holds *many* values for one record, which is the thing an array of strings /// is for - so the sentence points there rather than reporting an absence. CompositeType, - /// `Enum8(...)`, `Enum16(...)`. + /// An `ENUM` declared in a shape this dialect does not keep. /// - /// The storage an enum wants is what `MUTEX` already is: one value per record over an - /// interned dictionary. What is missing is the declared list of members, and nothing below - /// the catalog could enforce one. + /// **The type itself is answered now.** What is refused is the `'a' = 1` form, whose number + /// says which integer the value is stored as - here the dictionary assigns that, so keeping + /// the number would be a promise nothing honours - and a duplicate member, which would be + /// unreachable and would make `SHOW CREATE TABLE` disagree with what was typed. EnumType, /// `ARRAY JOIN`, `LATERAL VIEW explode(...)`, `UNNEST(...)`. /// @@ -1157,21 +1161,21 @@ impl Refused { language over them. Counting by period is `GROUP BY date_trunc(...)`" } Self::Sample => { - "a roaring container has no rank-select, so there is no way to take every tenth \ - matching record without decoding the nine in between - and the units that are \ - cheap to skip whole, a container and a shard, are not random subsets: record \ - ids are handed out in write order, so either one is a window of *time* wearing \ - the shape of a sample, and a number drawn from it would be biased in the \ - direction nobody checks. For fewer rows write `LIMIT`; for a defined subset \ - write a `WHERE` over a column that means something" + "`SAMPLE` takes one record in every `n`, written as `SAMPLE 10`, `SAMPLE 1/10` \ + or `SAMPLE 0.1` - three spellings of one stride. A fraction that is not one \ + over a whole number has no stride: `0.3` would be one in 3.33, and answering it \ + as one in three is a tenth more data than was asked for under the name that was \ + asked for. `SAMPLE BY` is a different thing again - it names the column a \ + sample is drawn on, and here the unit is the record id, which is not a column" } Self::Overwrite => { - "`TRUNCATE TABLE t` and then `INSERT INTO t SELECT ...` - two statements, \ - because they are two operations and nothing here wraps a pair of them in one \ - transaction. A single word that read as atomic and was not would be worse than \ - the two that admit it. To replace a table's contents *atomically*, write the \ - new rows into a second table and `EXCHANGE TABLES` them, which swaps two names \ - in one change and leaves the old rows under the other name" + "`INSERT OVERWRITE` is answered, but not at a partition: a shard here is \ + `record_id >> 20`, computed from the id a record was written under, so there is \ + nothing for `PARTITION (...)` to name. Write the statement without it. Note \ + what the plain form does and does not promise - it empties and then writes, and \ + nothing wraps those two in one transaction, so a crash between them leaves the \ + table empty. For a replacement that is genuinely atomic, write into a second \ + table and `EXCHANGE TABLES` them" } Self::CopyInto => { "loading data is `POST /table/{t}/import`, which streams rows in without a \ @@ -1211,10 +1215,13 @@ impl Refused { `Map` or a `Tuple` is two columns" } Self::EnumType => { - "`MUTEX` is the storage an enum is asking for: one value per record, interned \ - once into a dictionary, so a comparison is a bitmap and not a string. What is \ - not here is the declared list of members - nothing below the catalog would \ - check a value against one - so the column takes the values it is given" + "an enum is written `ENUM('a', 'b')` - the values, and nothing else. The \ + `'a' = 1` form says which integer each value is stored as, and here a mutex \ + interns each one into the dictionary and the *dictionary* decides that number, \ + so a written `= 1` would be a promise about storage that nothing keeps. A \ + repeated value is refused for a nearer reason: the second one is unreachable, \ + and a list holding fewer values than it names would make `SHOW CREATE TABLE` \ + disagree with what was typed" } Self::ArrayJoin => { "one row per element of a repeated column is `GROUP BY `, which is \ @@ -1390,10 +1397,11 @@ impl Refused { already carry: `FLOAT(10, 2)` is a `DECIMAL(10, 2)`, which keeps those digits \ exactly where a float would not. There is nothing here a blob or a JSON \ document lands in - a document is a `TEXT` column, and `JSONExtractString(c, \ - 'k')` reads a key out of one. Three more names map rather than missing: \ - `IPv4` is a `UINT(32)`, which is what an address is; `UUID` and `IPv6` are 128 \ - bits where a bit-sliced value stops at 64, so they are `TEXT` - interned once \ - and compared as bitmaps, but without the ordering a number would have" + 'k')` reads a key out of one. `IPv4` is not on this list at all: it is read as \ + the `UINT(32)` an address is. `UUID` and `IPv6` are, and the difference is what \ + would be lost - both are 128 bits where a bit-sliced value stops at 64, so the \ + only column they could land in is a keyed one, which answers `=` but not `<`. \ + Declare them `TEXT` if equality is all you need, and know that is what you have" } Self::Constraint => { "a column list here declares fields and nothing else: there are no rows for \ diff --git a/crates/big-sql/src/explain.rs b/crates/big-sql/src/explain.rs index 98d08f9..5865edc 100644 --- a/crates/big-sql/src/explain.rs +++ b/crates/big-sql/src/explain.rs @@ -621,18 +621,31 @@ fn qualified(database: &Option, table: &str) -> String { /// statement does. fn column(column: &Column) -> String { format!( - "{} {} depth={}{}", + "{} {} depth={}{}{}", column.name, column.kind.as_str(), column.bit_depth, - opt(" scale=", column.scale.as_ref()) + opt(" scale=", column.scale.as_ref()), + // **Printed, because the kind alone cannot tell an enum from the mutex it is stored + // as.** Without this an explained `ENUM('a','b')` and an explained `MUTEX` are the same + // line, and a corpus case about the members would be checking nothing. + match column.members.is_empty() { + true => String::new(), + false => format!( + " members=[{}]", + column.members.iter().map(|m| format!("'{m}'")).collect::>().join(", ") + ), + } ) } /// A write, as its columns and its rows. pub fn insert(insert: &Insert) -> String { let mut out = format!( - "Insert {} ({}){}", + "Insert{} {} ({}){}", + // Printed because it is the difference between writing rows and replacing every row - + // which is the one thing somebody explaining this statement most needs to see. + if insert.overwrite { " overwrite" } else { "" }, qualified(&insert.database, &insert.table), insert.columns.join(", "), // Which column is the record id is the whole difference between the two forms of this diff --git a/crates/big-sql/src/insert.rs b/crates/big-sql/src/insert.rs index edf7bec..854aef2 100644 --- a/crates/big-sql/src/insert.rs +++ b/crates/big-sql/src/insert.rs @@ -74,6 +74,18 @@ pub struct Insert { pub id_at: Option, /// Where the values come from: literals, or a query that reads them. pub source: Source, + /// `INSERT OVERWRITE`: empty the table first, then write these rows. + /// + /// **Two operations, and the statement says so rather than implying otherwise.** Nothing + /// here wraps a truncate and an insert in one transaction, so a crash between them leaves + /// the table empty - which is a real outcome and is documented rather than hidden. What it + /// buys over writing the two statements is that the source query is read *whole, before + /// anything is cleared*, so `INSERT OVERWRITE t (a) SELECT a FROM u` cannot half-empty `t` + /// because `u` turned out to be unreadable. + /// + /// For a replacement that is genuinely atomic, write into a second table and + /// `EXCHANGE TABLES` them. + pub overwrite: bool, } impl Insert { diff --git a/crates/big-sql/src/lib.rs b/crates/big-sql/src/lib.rs index 5f80be6..2f81058 100644 --- a/crates/big-sql/src/lib.rs +++ b/crates/big-sql/src/lib.rs @@ -217,6 +217,16 @@ impl Sql { Privilege::Insert, table_ref(insert.database.as_deref(), &insert.table), )); + // **An overwrite destroys what was there, so it demands the privilege that + // guards destroying.** The same argument `UPDATE` makes for demanding both: + // a credential that may add rows but not remove them is not most of the way to + // being allowed here - it is missing the half that empties the table. + if insert.overwrite { + out.push(Demand::new( + Privilege::Delete, + table_ref(insert.database.as_deref(), &insert.table), + )); + } // `INSERT ... SELECT` reads before it writes, and the read is a read: without // this the refusal would land after the coordinator had already taken a run of // record ids from the leader. diff --git a/crates/big-sql/src/lower/mod.rs b/crates/big-sql/src/lower/mod.rs index 0733e50..45dbb79 100644 --- a/crates/big-sql/src/lower/mod.rs +++ b/crates/big-sql/src/lower/mod.rs @@ -246,6 +246,17 @@ fn lower_one(select: &Select) -> Result { Some(cond) => rows(cond), None => call("All", vec![]), }; + // **Wrapped around the filter rather than beside it**, which is the order the two clauses + // mean: `WHERE amount > 5 SAMPLE 1/10` is a tenth of the records over five, not a tenth of + // every record of which some are over five. The second reading would answer a different + // question and would answer it with a smaller number, which is the kind of wrong nobody + // checks. + let rows = match select.sample { + None => rows, + Some(stride) => { + call("Sample", vec![rows, named("stride", Expr::Literal(Literal::Int(stride.into())))]) + } + }; // Three buckets, because the select list decides the plans and every legal list is one of a // small number of combinations of them. diff --git a/crates/big-sql/src/parse/create.rs b/crates/big-sql/src/parse/create.rs index a71e3ad..949f691 100644 --- a/crates/big-sql/src/parse/create.rs +++ b/crates/big-sql/src/parse/create.rs @@ -275,20 +275,40 @@ impl Parser<'_> { // width in brackets is refused below rather than read as a depth: `INT(11)` is eleven // digits where MySQL wrote it and eleven bits here, and eleven bits stop at 2047. let column = match word.as_str() { - "SET" => Column { name, kind: Set, bit_depth: KEYLESS_DEPTH, scale: None }, - "MUTEX" => Column { name, kind: Mutex, bit_depth: KEYLESS_DEPTH, scale: None }, - "BOOL" | "BOOLEAN" => { - Column { name, kind: Bool, bit_depth: KEYLESS_DEPTH, scale: None } - } + "SET" => Column { + name, + kind: Set, + bit_depth: KEYLESS_DEPTH, + scale: None, + members: Vec::new(), + }, + "MUTEX" => Column { + name, + kind: Mutex, + bit_depth: KEYLESS_DEPTH, + scale: None, + members: Vec::new(), + }, + "BOOL" | "BOOLEAN" => Column { + name, + kind: Bool, + bit_depth: KEYLESS_DEPTH, + scale: None, + members: Vec::new(), + }, // A time quantum is keyed by a moment and viewed by day, and it keeps its own // spelling. `DATE` and `DATETIME` used to be spellings of it too, and stopped being // when they got scalar kinds: a keyed field answers `f = 'k' AND f BETWEEN lo AND // hi` and nothing else, so `WHERE d >= '2024-01-01'`, `ORDER BY d` and `max(d)` - // which is most of what anyone writes a date column for - had no way to be asked. // The two are different things and now have different names. - "TIMEQUANTUM" => { - Column { name, kind: TimeQuantum, bit_depth: KEYLESS_DEPTH, scale: None } - } + "TIMEQUANTUM" => Column { + name, + kind: TimeQuantum, + bit_depth: KEYLESS_DEPTH, + scale: None, + members: Vec::new(), + }, // A day count and a second count, both from 1970 and both signed, because dates // before it are ordinary values. The depths are what those counts need: 32 planes of // days is ±5.8 million years, and seconds want the full width. @@ -314,7 +334,7 @@ impl Parser<'_> { "FLOAT" | "FLOAT32" | "REAL" => (Float32, 32), _ => (Float64, 64), }; - Column { name, kind, bit_depth, scale: None } + Column { name, kind, bit_depth, scale: None, members: Vec::new() } } // A length on a key is a bound on nothing: keys are stored whole, and there is no // truncation or padding for a `VARCHAR(255)` to describe. @@ -322,7 +342,13 @@ impl Parser<'_> { if self.peek() == Some(&Tok::LParen) { return Err(self.refuse(Refused::Constraint)); } - Column { name, kind: Set, bit_depth: KEYLESS_DEPTH, scale: None } + Column { + name, + kind: Set, + bit_depth: KEYLESS_DEPTH, + scale: None, + members: Vec::new(), + } } // **The one type name that describes an encoding rather than a domain**, and the // encoding it describes is the only one a keyed field has: a `SET` interns each @@ -340,7 +366,13 @@ impl Parser<'_> { if !matches!(inner.as_str(), "STRING" | "TEXT" | "VARCHAR" | "CHAR") { return Err(self.refuse_at(Refused::ColumnType, at)); } - Column { name, kind: Set, bit_depth: KEYLESS_DEPTH, scale: None } + Column { + name, + kind: Set, + bit_depth: KEYLESS_DEPTH, + scale: None, + members: Vec::new(), + } } // The two wrappers this engine recognises and declines, refused at the word that // names them rather than after the bracket is read. Nothing is consumed first @@ -355,9 +387,10 @@ impl Parser<'_> { "ARRAY" | "MAP" | "TUPLE" | "NESTED" => { return Err(self.refuse_at(Refused::CompositeType, at)) } - // Its own sentence rather than the composite one, because what it needs is a - // different rewrite: `MUTEX` is already the storage an enum wants. - "ENUM" | "ENUM8" | "ENUM16" => return Err(self.refuse_at(Refused::EnumType, at)), + // `ENUM('a', 'b')`: a mutex that also says which values it may hold. The width in + // `Enum8`/`Enum16` is read and discarded, the way `INT UNSIGNED`'s word is: it + // names how many members fit, and the dictionary underneath has no such ceiling. + "ENUM" | "ENUM8" | "ENUM16" => self.enum_members(name)?, "BITMAP" | "AGGREGATEFUNCTION" => return Err(self.refuse_at(Refused::BitmapType, at)), // The sketch types, which are a different refusal from `BITMAP` even though both // decline a column: a bitmap is refused because every column here already is one, @@ -384,16 +417,32 @@ impl Parser<'_> { self.eat_word("UNSIGNED"); Int }; - Column { name, kind, bit_depth, scale: None } + Column { name, kind, bit_depth, scale: None, members: Vec::new() } } // The two native spellings, which exist because the SQL names carry a fixed width // and a bit-sliced field is cheaper the narrower it is: every plane is one more // bitmap a range query intersects. - "UINT" => Column { name, kind: Int, bit_depth: self.bit_depth()?, scale: None }, + // **An IPv4 address is a 32-bit number, and nothing is lost by saying so.** It + // compares, sorts, ranges and zone-maps exactly as `UINT(32)` does, because that is + // what it is - so this joins `TIMESTAMP` -> `DATETIME` and `BIGINT` -> `UINT(64)` + // as a spelling this dialect reads and writes back under its canonical name. + // + // `UUID` and `IPv6` are deliberately *not* here. Both are 128 bits where a + // bit-sliced value stops at 64, so the only column they could land in is a keyed + // one - and a keyed column answers `=` but not `<`. Accepting them would take an + // ordering away silently; refusing them says so (`sql_unknown_column_type`). + "IPV4" => Column { name, kind: Int, bit_depth: 32, scale: None, members: Vec::new() }, + "UINT" => Column { + name, + kind: Int, + bit_depth: self.bit_depth()?, + scale: None, + members: Vec::new(), + }, "SIGNED" => { let bit_depth = if self.peek() == Some(&Tok::LParen) { self.bit_depth()? } else { 32 }; - Column { name, kind: Signed, bit_depth, scale: None } + Column { name, kind: Signed, bit_depth, scale: None, members: Vec::new() } } "DECIMAL" | "NUMERIC" => self.decimal(name)?, _ => return Err(self.refuse_at(Refused::ColumnType, at)), @@ -414,6 +463,57 @@ impl Parser<'_> { Ok(column) } + /// `ENUM('a', 'b')`, with the type name consumed. + /// + /// # Why the numbers are refused + /// + /// `Enum8('a' = 1, 'b' = 2)` is what ClickHouse writes, and the number says which integer + /// the value is stored as. Here a mutex interns each value into the dictionary and the + /// *dictionary* decides that number, so an accepted `= 1` would be a promise about storage + /// that nothing keeps. Refused at the `=` with the spelling that works, rather than read and + /// dropped - a number silently ignored is the failure this whole surface is built to avoid. + /// + /// Duplicates are refused for the same reason a rename onto a live name is: the second one + /// would be unreachable, and a list that quietly held fewer values than it named would make + /// `SHOW CREATE TABLE` disagree with what was typed. + fn enum_members(&mut self, name: String) -> Result { + let at = self.at(); + if !self.eat(&Tok::LParen) { + return Err(self.syntax("( and the values the enum may hold")); + } + let mut members: Vec = Vec::new(); + loop { + let value = match self.peek() { + Some(Tok::Str(s)) => { + let s = s.clone(); + self.i += 1; + s + } + _ => return Err(self.syntax("a quoted value")), + }; + // The one byte a member may not hold, because it is what separates them on disk. + if value.contains('\0') || members.contains(&value) { + return Err(self.refuse_at(Refused::EnumType, at)); + } + members.push(value); + // `= 1`, which promises a storage number the dictionary assigns. + if self.peek() == Some(&Tok::Op("=")) { + return Err(self.refuse_at(Refused::EnumType, at)); + } + if !self.eat(&Tok::Comma) { + break; + } + } + if !self.eat(&Tok::RParen) { + return Err(self.syntax(") after the values")); + } + // An enum of nothing is a column that can hold nothing, which is a table nobody meant. + if members.is_empty() { + return Err(self.refuse_at(Refused::EnumType, at)); + } + Ok(Column { name, kind: ColumnKind::Enum, bit_depth: KEYLESS_DEPTH, scale: None, members }) + } + /// `DECIMAL(precision, scale)`, with the type name consumed. /// /// Both numbers, always. `DECIMAL(10)` is SQL for ten digits and no fraction, and reading @@ -452,6 +552,7 @@ impl Parser<'_> { kind: ColumnKind::Decimal, bit_depth: bits as u32, scale: Some(scale as i8), + members: Vec::new(), }) } diff --git a/crates/big-sql/src/parse/insert.rs b/crates/big-sql/src/parse/insert.rs index a20c6b4..41f5d64 100644 --- a/crates/big-sql/src/parse/insert.rs +++ b/crates/big-sql/src/parse/insert.rs @@ -29,15 +29,23 @@ impl Parser<'_> { /// that caused it: a missing column list at the table name, a missing `id` at the closing /// bracket of the list that should have held one, and a `SELECT` at the `SELECT`. pub(super) fn insert(&mut self) -> Result { - // Before `INTO`, because `INSERT OVERWRITE TABLE t` writes neither word in the place - // this expects one. It is a truncate and an insert, and nothing wraps two of those in - // one transaction - so the sentence offers the two statements, and the atomic spelling. - if self.word_is("OVERWRITE") { - return Err(self.refuse(Refused::Overwrite)); + // `INSERT OVERWRITE [TABLE] t (...)`: empty the table, then write. `TABLE` is optional + // here for the reason it is optional after `TRUNCATE` - Hive and Spark write it, and the + // word decides nothing. + let overwrite = self.eat_word("OVERWRITE"); + if overwrite { + self.eat_word("TABLE"); } self.eat_word("INTO"); let (database, table) = self.table_ref("a table name")?; + // `PARTITION (dt = '...')` is the half of `INSERT OVERWRITE` that stays refused: there + // are no partitions here to aim one at, because a shard is a function of the record id. + // Narrowed to this from refusing the whole statement, which is a widening. + if self.word_is("PARTITION") { + return Err(self.refuse(Refused::Overwrite)); + } + // `INSERT INTO t SELECT ...` with no column list would be positional against a field // order the statement does not carry, which is the same refusal `VALUES` earns. if self.word_is("SELECT") || self.word_is("WITH") { @@ -67,7 +75,7 @@ impl Parser<'_> { if self.peek().is_some() { return Err(self.syntax("the end of the statement")); } - return Ok(Insert { database, table, columns, id_at, source }); + return Ok(Insert { database, table, columns, id_at, source, overwrite }); } // `VALUE` is MySQL's spelling of the same word and means the same thing. @@ -91,7 +99,7 @@ impl Parser<'_> { if self.peek().is_some() { return Err(self.syntax("the end of the statement")); } - Ok(Insert { database, table, columns, id_at, source: Source::Values(rows) }) + Ok(Insert { database, table, columns, id_at, source: Source::Values(rows), overwrite }) } /// The `SELECT` an `INSERT` reads its values from, with `SELECT` still ahead. diff --git a/crates/big-sql/src/parse/select.rs b/crates/big-sql/src/parse/select.rs index a19bdda..115d710 100644 --- a/crates/big-sql/src/parse/select.rs +++ b/crates/big-sql/src/parse/select.rs @@ -76,12 +76,10 @@ impl Parser<'_> { } let joins = self.joins()?; - // `FROM t SAMPLE 0.1`. Refused at the word, after the joins so that `SAMPLE` on either - // side of one lands here rather than as "expected WHERE". `at_clause_keyword` is what - // stops it being eaten as the table's alias before this is reached. - if self.word_is("SAMPLE") { - return Err(self.refuse(Refused::Sample)); - } + // `FROM t SAMPLE 0.1`, read after the joins so it belongs to the statement rather than + // to whichever table it happened to follow. `at_clause_keyword` is what stops it being + // eaten as the table's alias before this is reached. + let sample = if self.eat_word("SAMPLE") { Some(self.sample_stride()?) } else { None }; // `PREWHERE` is ClickHouse's hint to filter on a cheap column before reading the rest // of the row. **Here it selects exactly the set `WHERE` would**, because there is no @@ -241,6 +239,7 @@ impl Parser<'_> { let format = self.format()?; Ok(Select { + sample, items, from, joins, @@ -280,6 +279,52 @@ impl Parser<'_> { Ok(Source { database, table, alias }) } + /// The `n` of `SAMPLE`, however it was written: `1/10`, `0.1` and `10` are one stride. + /// + /// **Three spellings, and each is exact.** `SAMPLE 10` is one record in ten. `SAMPLE 1/10` + /// says the same thing the way a ratio is written. `SAMPLE 0.1` is ClickHouse's spelling and + /// is read through the decimal it already parsed as - `units` and `scale`, not a float - so + /// `0.1` is exactly one in ten rather than one in 9.999999. + /// + /// A fraction that is not one over a whole number is refused rather than rounded. `SAMPLE + /// 0.3` would be one in 3.33, and answering it as one in three is a tenth more data than was + /// asked for under the name that was asked for. + fn sample_stride(&mut self) -> Result { + let at = self.at(); + let bad = |p: &Self| p.refuse_at(Refused::Sample, at); + + // `SAMPLE 1/10`: a ratio, and the numerator has to be one - `2/10` is a fraction this + // would have to reduce, and a reduction is a second place for it to disagree with what + // was written. + if let Some(Tok::Num(Literal::Int(1))) = self.peek() { + if self.tok_at_is(1, &Tok::Arith("/")) { + self.i += 2; + let n = self.small_number("how many records to take one of")?; + return u32::try_from(n).ok().filter(|n| *n >= 2).ok_or_else(|| bad(self)); + } + } + match self.peek() { + // `SAMPLE 10`. + Some(Tok::Num(Literal::Int(n))) => { + let n = *n; + self.i += 1; + u32::try_from(n).ok().filter(|n| *n >= 2).ok_or_else(|| bad(self)) + } + // `SAMPLE 0.1`. Exact, because a decimal here is an integer and a scale: the stride + // is `10^scale / units`, and it exists only when that divides. + Some(Tok::Num(Literal::Dec { units, scale })) => { + let (units, scale) = (*units, *scale); + self.i += 1; + let whole = 10u64.checked_pow(u32::from(scale)).ok_or_else(|| bad(self))?; + if units == 0 || whole % units != 0 { + return Err(bad(self)); + } + u32::try_from(whole / units).ok().filter(|n| *n >= 2).ok_or_else(|| bad(self)) + } + _ => Err(self.syntax("how much to sample")), + } + } + /// Whether the current word begins a clause rather than being an alias. pub(super) fn at_clause_keyword(&self) -> bool { const KEYWORDS: [&str; 25] = [ diff --git a/crates/big-sql/src/render.rs b/crates/big-sql/src/render.rs index 3619328..b3ab05e 100644 --- a/crates/big-sql/src/render.rs +++ b/crates/big-sql/src/render.rs @@ -62,6 +62,18 @@ pub fn create_table(table: &str, engine: Option<&str>, columns: &[Column]) -> St /// One column's type, in the spelling `Parser::column_type` reads back as this same column. fn column_type(column: &Column) -> String { match column.kind { + // The declared type rather than the storage one, which is the whole point of keeping the + // members: a column written as an enum reads back as one. Quoted with `''` doubled, the + // way every string literal in this dialect is written. + ColumnKind::Enum => format!( + "ENUM({})", + column + .members + .iter() + .map(|m| format!("'{}'", m.replace('\'', "''"))) + .collect::>() + .join(", ") + ), ColumnKind::Set => "SET".to_string(), ColumnKind::Mutex => "MUTEX".to_string(), ColumnKind::Bool => "BOOL".to_string(), diff --git a/crates/big-sql/tests/testdata/ddl.test b/crates/big-sql/tests/testdata/ddl.test index 7e09642..43e21e7 100644 --- a/crates/big-sql/tests/testdata/ddl.test +++ b/crates/big-sql/tests/testdata/ddl.test @@ -389,3 +389,76 @@ error CREATE TABLE t (q QUANTILE_STATE) ---- sql_sketch + +# `IPv4` is a 32-bit number, so it is read as one - the same way `TIMESTAMP` is read as +# `DATETIME` and `BIGINT` as `UINT(64)`. Nothing is lost: it compares, sorts and zone-maps as +# what it is, and it writes back under the canonical spelling. +ddl +CREATE TABLE t (client IPv4) +---- +CreateTable t +└── client int depth=32 + +# `UUID` and `IPv6` are not, and the difference is what would be lost. Both are 128 bits where a +# bit-sliced value stops at 64, so the only column they could land in is keyed - which answers +# `=` but not `<`. Refused rather than accepted with the ordering quietly gone. +error +CREATE TABLE t (id UUID) +---- +sql_unknown_column_type + +error +CREATE TABLE t (client IPv6) +---- +sql_unknown_column_type + +## `ENUM`: a `MUTEX` that also declares which values it may hold. +# +# The kind stored is a mutex — one value per record over an interned dictionary, which is exactly +# what an enum wants — and the member list travels beside it as metadata. That is what makes this +# free below the catalog: no new `FieldKind`, no engine change, no storage format for a value. +# +# What the list buys is the two things a mutex cannot say: a write outside it is refused, and the +# declared type survives `SHOW CREATE TABLE` instead of coming back as `MUTEX`. + +ddl +CREATE TABLE t (status ENUM('new', 'paid', 'shipped')) +---- +CreateTable t +└── status mutex depth=32 members=['new', 'paid', 'shipped'] + +# `Enum8` and `Enum16` are the same declaration. The width names how many members fit, and the +# dictionary underneath has no such ceiling — so it is read and dropped, the way `INT UNSIGNED`'s +# word is. +ddl +CREATE TABLE t (status Enum8('a', 'b')) +---- +CreateTable t +└── status mutex depth=32 members=['a', 'b'] + +ddl +CREATE TABLE t (status Enum16('a')) +---- +CreateTable t +└── status mutex depth=32 members=['a'] + +# **The numbers are refused, not ignored.** `Enum8('a' = 1)` says which integer the value is +# stored as, and here the dictionary assigns that — so an accepted `= 1` would be a promise about +# storage that nothing keeps. +error +CREATE TABLE t (status Enum8('a' = 1, 'b' = 2)) +---- +sql_enum_type + +# A duplicate member would be unreachable, and would make `SHOW CREATE TABLE` disagree with what +# was typed. +error +CREATE TABLE t (status ENUM('a', 'a')) +---- +sql_enum_type + +# An enum of nothing is a column that can hold nothing. +error +CREATE TABLE t (status ENUM()) +---- +parse_error diff --git a/crates/big-sql/tests/testdata/insert.test b/crates/big-sql/tests/testdata/insert.test index 9888ed8..669d7c5 100644 --- a/crates/big-sql/tests/testdata/insert.test +++ b/crates/big-sql/tests/testdata/insert.test @@ -140,3 +140,38 @@ INSERT INTO tx (country, amount) SELECT country, sum(amount) FROM u GROUP BY cou ---- Insert tx (country, amount) id=allocated └── select country, sum from u + +## `INSERT OVERWRITE`: empty the table, then write. +# +# Two operations with no transaction around them, and the statement says so rather than implying +# otherwise. What it buys over writing `TRUNCATE` and `INSERT` separately is the *order*: the +# source query is read whole and merged before anything is cleared, so a `SELECT` that turns out +# to be unreadable leaves the table alone. The atomic replacement is `EXCHANGE TABLES`. + +insert +INSERT OVERWRITE INTO t (amount) VALUES (1), (2) +---- +Insert overwrite t (amount) id=allocated +├── 1 +└── 2 + +# `TABLE` is optional, the way it is after `TRUNCATE` - Hive and Spark write it, and the word +# decides nothing. +insert +INSERT OVERWRITE TABLE t (amount) VALUES (1) +---- +Insert overwrite t (amount) id=allocated +└── 1 + +insert +INSERT OVERWRITE t (amount) SELECT amount FROM u +---- +Insert overwrite t (amount) id=allocated +└── select amount from u + +# Without the word it is the ordinary statement, and nothing is emptied. +insert +INSERT INTO t (amount) VALUES (1) +---- +Insert t (amount) id=allocated +└── 1 diff --git a/crates/big-sql/tests/testdata/refusals.test b/crates/big-sql/tests/testdata/refusals.test index 639cc23..3494a60 100644 --- a/crates/big-sql/tests/testdata/refusals.test +++ b/crates/big-sql/tests/testdata/refusals.test @@ -1203,33 +1203,47 @@ sql_no_optimize # the local spelling for that question - not the news that a feature is absent. Where there is no # local spelling, it says which decision the absence follows from. -## Sampling. There is no rank-select over a container, and the units that *are* cheap to skip -## whole are windows of write order rather than random subsets. +## Sampling. `SAMPLE` is answered - see `select.test` - but only where the fraction names a +## whole stride, and only on a statement rather than on a column. +# One in 3.33 is not a stride, and answering it as one in three would be a tenth more data than +# was asked for under the name that was asked for. error -SELECT count(*) FROM t SAMPLE 0.1 +SELECT count(*) FROM t SAMPLE 0.3 ---- sql_no_sample error -SELECT count(*) FROM t SAMPLE 0.1 WHERE amount > 5 +SELECT count(*) FROM t SAMPLE 0.15 ---- sql_no_sample +# Taking every record is what the statement without the clause already says. error -CREATE TABLE t (a SET) SAMPLE BY a +SELECT count(*) FROM t SAMPLE 1 ---- sql_no_sample -## Writes that are two operations wearing one word. +# A ratio has to have one on top: `2/10` is a fraction this would have to reduce, and a +# reduction is a second place for it to disagree with what was written. +error +SELECT count(*) FROM t SAMPLE 2/10 +---- +parse_error +# `SAMPLE BY` declares which column a sample is *drawn on*, and here the unit is the record id - +# which is not a column and cannot be chosen. error -INSERT OVERWRITE TABLE t SELECT a FROM u +CREATE TABLE t (a SET) SAMPLE BY a ---- -sql_no_overwrite +sql_no_sample + +## Writes that are two operations wearing one word. +# `INSERT OVERWRITE` is answered now; what stays refused is aiming it at a partition, because a +# shard is a function of the record id and there is nothing for the clause to name. error -INSERT OVERWRITE t VALUES (1) +INSERT OVERWRITE TABLE t PARTITION (dt = '2026-01-01') SELECT amount FROM u ---- sql_no_overwrite diff --git a/crates/big-sql/tests/testdata/select.test b/crates/big-sql/tests/testdata/select.test index 3ce3d81..9b6d7fc 100644 --- a/crates/big-sql/tests/testdata/select.test +++ b/crates/big-sql/tests/testdata/select.test @@ -312,3 +312,50 @@ Row ├── count = #0 ├── sum = #1 └── having: #0 > 1000 and #1 > 500 + +## `SAMPLE`: one record in every n, by record id. +# +# **The unit is the id's low bits, not a block of them**, and that is what makes it a sample +# rather than a slice: ids are handed out in write order, so every tenth *container* would be +# every tenth window of time. Every tenth *id* is spread evenly through each of those windows. +# +# It wraps the filter rather than sitting beside it, because that is the order the two clauses +# mean: a tenth of the records over five, not a tenth of every record of which some are over +# five. + +plan +SELECT count(*) FROM t SAMPLE 10 +---- +Count t +└── sample 1 in 10 + └── all + +# Three spellings, one stride. `0.1` is read through the decimal it parsed as - units and scale, +# never a float - so it is exactly one in ten. +plan +SELECT count(*) FROM t SAMPLE 1/10 +---- +Count t +└── sample 1 in 10 + └── all + +plan +SELECT count(*) FROM t SAMPLE 0.1 +---- +Count t +└── sample 1 in 10 + └── all + +plan +SELECT count(*) FROM t SAMPLE 0.5 WHERE amount > 5 +---- +Count t +└── sample 1 in 2 + └── amount > 5 + +shape +SELECT country, count(*) FROM t SAMPLE 4 GROUP BY country +---- +Groups keys=[#0] +├── country = key +└── count = group #0 else 0 diff --git a/crates/big-sql/tests/translate/introspect.rs b/crates/big-sql/tests/translate/introspect.rs index 80573da..039c76c 100644 --- a/crates/big-sql/tests/translate/introspect.rs +++ b/crates/big-sql/tests/translate/introspect.rs @@ -119,6 +119,7 @@ fn a_decimal_that_came_from_the_field_route_renders_wide_enough() { kind: ColumnKind::Decimal, bit_depth: 20, scale: Some(2), + members: Vec::new(), }; let written = big_sql::render::create_table("t", None, &[column]); let read_back = columns(&written); diff --git a/crates/big-sql/tests/translate/schema.rs b/crates/big-sql/tests/translate/schema.rs index e6b9677..8e668d9 100644 --- a/crates/big-sql/tests/translate/schema.rs +++ b/crates/big-sql/tests/translate/schema.rs @@ -282,6 +282,7 @@ fn alter_table_adds_and_drops_fields() { kind: ColumnKind::Set, bit_depth: 32, scale: None, + members: Vec::new(), })], } );