diff --git a/crates/sprout-core/src/kind.rs b/crates/sprout-core/src/kind.rs index 3e317586..164b48ae 100644 --- a/crates/sprout-core/src/kind.rs +++ b/crates/sprout-core/src/kind.rs @@ -21,6 +21,10 @@ pub const KIND_REACTION: u32 = 7; pub const KIND_GIFT_WRAP: u32 = 1059; /// NIP-94: File metadata attachment. pub const KIND_FILE_METADATA: u32 = 1063; +/// NIP-23: Long-form content (articles, blog posts, RFCs). +/// Parameterized replaceable (NIP-33, 30000–39999 range) — keyed by `(pubkey, kind, d_tag)`. +/// Stored globally (channel_id = NULL); author-owned, not channel-scoped. +pub const KIND_LONG_FORM: u32 = 30023; /// NIP-42 auth event — never stored (carries bearer tokens). pub const KIND_AUTH: u32 = 22242; @@ -62,6 +66,11 @@ pub const KIND_NIP29_GROUP_MEMBERS: u32 = 39002; /// NIP-29: Addressable group roles definition. pub const KIND_NIP29_GROUP_ROLES: u32 = 39003; +/// Lower bound of the NIP-33 parameterized replaceable range (30000–39999). +pub const PARAM_REPLACEABLE_KIND_MIN: u32 = 30000; +/// Upper bound of the NIP-33 parameterized replaceable range (30000–39999). +pub const PARAM_REPLACEABLE_KIND_MAX: u32 = 39999; + /// Lower bound of the ephemeral event range (20000–29999). Never stored. pub const EPHEMERAL_KIND_MIN: u32 = 20000; /// Upper bound of the ephemeral event range (20000–29999). Never stored. @@ -271,6 +280,7 @@ pub const ALL_KINDS: &[u32] = &[ KIND_SUBSCRIPTION_RESUMED, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, + KIND_LONG_FORM, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_FORUM_COMMENT, @@ -308,11 +318,18 @@ pub const fn is_ephemeral(kind: u32) -> bool { /// Returns `true` if `kind` is replaceable (NIP-01: kinds 0, 3, 41, 10000–19999). /// NIP-33 parameterized-replaceable kinds (30000–39999) use a different replacement -/// key (includes `d`-tag) and are handled separately via `replace_addressable_event`. +/// key (includes `d`-tag) and are handled separately via `replace_parameterized_event`. pub const fn is_replaceable(kind: u32) -> bool { matches!(kind, 0 | 3 | KIND_CHANNEL_METADATA | 10000..=19999) } +/// Returns `true` if `kind` is in the NIP-33 parameterized replaceable range (30000–39999). +/// +/// These events are keyed by `(pubkey, kind, d_tag)` — the latest `created_at` wins. +pub const fn is_parameterized_replaceable(kind: u32) -> bool { + kind >= PARAM_REPLACEABLE_KIND_MIN && kind <= PARAM_REPLACEABLE_KIND_MAX +} + /// Returns `true` if `kind` is a workflow execution event (46001–46012). /// These must not trigger workflows (prevents infinite loops). pub const fn is_workflow_execution_kind(kind: u32) -> bool { @@ -348,4 +365,24 @@ mod tests { assert!(seen.insert(k), "duplicate kind value: {k}"); } } + + #[test] + fn parameterized_replaceable_range() { + assert!(!is_parameterized_replaceable(29999)); + assert!(is_parameterized_replaceable(30000)); + assert!(is_parameterized_replaceable(30023)); // NIP-23 long-form + assert!(is_parameterized_replaceable(39000)); // NIP-29 group metadata + assert!(is_parameterized_replaceable(39999)); + assert!(!is_parameterized_replaceable(40000)); + } + + #[test] + fn replaceable_and_parameterized_are_disjoint() { + for kind in 0..=65535u32 { + assert!( + !(is_replaceable(kind) && is_parameterized_replaceable(kind)), + "kind {kind} is both replaceable and parameterized replaceable" + ); + } + } } diff --git a/crates/sprout-db/src/event.rs b/crates/sprout-db/src/event.rs index b7cf9b12..0559d7b0 100644 --- a/crates/sprout-db/src/event.rs +++ b/crates/sprout-db/src/event.rs @@ -9,7 +9,7 @@ use nostr::Event; use sqlx::{PgPool, QueryBuilder, Row}; use uuid::Uuid; -use sprout_core::kind::{event_kind_i32, is_ephemeral, KIND_AUTH}; +use sprout_core::kind::{event_kind_i32, is_ephemeral, is_parameterized_replaceable, KIND_AUTH}; use sprout_core::StoredEvent; use crate::error::{DbError, Result}; @@ -34,6 +34,38 @@ pub struct EventQuery { /// Restrict to events with a `p` tag mentioning this hex pubkey. /// Joins against `event_mentions` table (indexed). pub p_tag_hex: Option, + /// Restrict to events with this exact `d_tag` value (NIP-33). + /// Pushed into SQL via the `idx_events_parameterized` index. + pub d_tag: Option, +} + +/// Maximum length for a `d_tag` value (bytes). NIP-33 d-tags are short identifiers; +/// anything beyond this is either a bug or abuse. +pub const D_TAG_MAX_LEN: usize = 1024; + +/// Extract the `d_tag` value for storage. +/// +/// For NIP-33 parameterized replaceable events (kind 30000–39999): returns the first +/// `d` tag's value, or `""` if no `d` tag is present (per NIP-33 spec). +/// For all other events: returns `None` (column stays NULL). +pub fn extract_d_tag(event: &Event) -> Option { + let kind_u32 = event.kind.as_u16() as u32; + if !is_parameterized_replaceable(kind_u32) { + return None; + } + let val = event + .tags + .iter() + .find_map(|tag| { + let parts = tag.as_slice(); + if parts.len() >= 2 && parts[0] == "d" { + Some(parts[1].to_string()) + } else { + None + } + }) + .unwrap_or_default(); // Missing d tag → empty string per NIP-33 + Some(val) } /// Insert a Nostr event. Rejects AUTH and ephemeral kinds. @@ -64,10 +96,11 @@ pub async fn insert_event( let created_at = DateTime::from_timestamp(created_at_secs, 0) .ok_or(DbError::InvalidTimestamp(created_at_secs))?; let received_at = Utc::now(); + let d_tag = extract_d_tag(event); let result = sqlx::query( r#" - INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT DO NOTHING "#, ) @@ -80,6 +113,7 @@ pub async fn insert_event( .bind(sig_bytes.as_slice()) .bind(received_at) .bind(channel_id) + .bind(d_tag.as_deref()) .execute(pool) .await?; @@ -152,6 +186,11 @@ pub async fn query_events(pool: &PgPool, q: &EventQuery) -> Result) -> nostr::Event { + let keys = Keys::generate(); + EventBuilder::new(Kind::Custom(kind), "test", tags) + .sign_with_keys(&keys) + .expect("sign") + } + + #[test] + fn extract_d_tag_from_nip33_event() { + let event = make_event_with_kind_and_tags( + 30023, + vec![Tag::parse(&["d", "my-article-slug"]).unwrap()], + ); + assert_eq!(extract_d_tag(&event), Some("my-article-slug".to_string())); + } + + #[test] + fn extract_d_tag_first_d_wins() { + let event = make_event_with_kind_and_tags( + 30023, + vec![ + Tag::parse(&["d", "first"]).unwrap(), + Tag::parse(&["d", "second"]).unwrap(), + ], + ); + assert_eq!(extract_d_tag(&event), Some("first".to_string())); + } + + #[test] + fn extract_d_tag_missing_becomes_empty_string() { + // NIP-33: "if there is no d tag, the d tag is considered to be ''" + let event = + make_event_with_kind_and_tags(30023, vec![Tag::parse(&["p", "abc123"]).unwrap()]); + assert_eq!(extract_d_tag(&event), Some(String::new())); + } + + #[test] + fn extract_d_tag_empty_value_preserved() { + let event = make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d", ""]).unwrap()]); + assert_eq!(extract_d_tag(&event), Some(String::new())); + } + + #[test] + fn extract_d_tag_non_nip33_returns_none() { + // kind:1 (text note) — not parameterized replaceable + let event = make_event_with_kind_and_tags( + 1, + vec![Tag::parse(&["d", "should-be-ignored"]).unwrap()], + ); + assert_eq!(extract_d_tag(&event), None); + } + + #[test] + fn extract_d_tag_nip29_group_metadata() { + // kind:39000 is in the 30000–39999 range — d_tag should be extracted + let event = + make_event_with_kind_and_tags(39000, vec![Tag::parse(&["d", "group-id"]).unwrap()]); + assert_eq!(extract_d_tag(&event), Some("group-id".to_string())); + } + + #[test] + fn extract_d_tag_boundary_kinds() { + // kind:29999 — just below range + let below = make_event_with_kind_and_tags(29999, vec![Tag::parse(&["d", "val"]).unwrap()]); + assert_eq!(extract_d_tag(&below), None); + + // kind:30000 — lower bound + let lower = make_event_with_kind_and_tags(30000, vec![Tag::parse(&["d", "val"]).unwrap()]); + assert_eq!(extract_d_tag(&lower), Some("val".to_string())); + + // kind:39999 — upper bound + let upper = make_event_with_kind_and_tags(39999, vec![Tag::parse(&["d", "val"]).unwrap()]); + assert_eq!(extract_d_tag(&upper), Some("val".to_string())); + + // kind:40000 — just above range + let above = make_event_with_kind_and_tags(40000, vec![Tag::parse(&["d", "val"]).unwrap()]); + assert_eq!(extract_d_tag(&above), None); + } + + #[test] + fn extract_d_tag_single_element_d_tag_ignored() { + // A d tag with only one element (no value) should not match — parts.len() < 2 + let event = make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d"]).unwrap()]); + // No d tag with a value → empty string per NIP-33 + assert_eq!(extract_d_tag(&event), Some(String::new())); + } + + #[test] + fn extract_d_tag_preserves_full_value() { + // extract_d_tag returns the full value — length enforcement is at the ingest layer. + let long_val = "x".repeat(2048); + let event = + make_event_with_kind_and_tags(30023, vec![Tag::parse(&["d", &long_val]).unwrap()]); + let result = extract_d_tag(&event).unwrap(); + assert_eq!(result.len(), 2048); + assert_eq!(result, long_val); + } +} diff --git a/crates/sprout-db/src/lib.rs b/crates/sprout-db/src/lib.rs index a3b64000..359a2bdb 100644 --- a/crates/sprout-db/src/lib.rs +++ b/crates/sprout-db/src/lib.rs @@ -1179,6 +1179,25 @@ impl Db { partition::ensure_future_partitions(&self.pool, months_ahead).await } + /// Backfill `d_tag` for existing NIP-33 events (kind 30000–39999) that have `d_tag IS NULL`. + /// + /// Idempotent — safe to call on every startup. No-ops when all rows are already populated. + /// Runs a single UPDATE touching only NIP-33 rows with NULL d_tag. + pub async fn backfill_d_tags(&self) -> Result { + let result = sqlx::query( + "UPDATE events \ + SET d_tag = COALESCE( \ + (SELECT elem->>1 FROM jsonb_array_elements(tags) AS elem \ + WHERE elem->>0 = 'd' LIMIT 1), \ + '' \ + ) \ + WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL", + ) + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } + // ── Pubkey Allowlist ───────────────────────────────────────────────────── /// Check if a pubkey is in the allowlist. @@ -1367,10 +1386,11 @@ impl Db { let sig_bytes = event.sig.serialize(); let tags_json = serde_json::to_value(&event.tags)?; let received_at = chrono::Utc::now(); + let d_tag = crate::event::extract_d_tag(event); let insert_result = sqlx::query( - "INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) \ + "INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ ON CONFLICT DO NOTHING", ) .bind(event.id.as_bytes().as_slice()) @@ -1382,6 +1402,7 @@ impl Db { .bind(sig_bytes.as_slice()) .bind(received_at) .bind(channel_id) + .bind(d_tag.as_deref()) .execute(&mut *tx) .await?; @@ -1409,6 +1430,144 @@ impl Db { true, )) } + + /// Atomically replace a NIP-33 parameterized replaceable event (kind 30000–39999). + /// + /// Keeps only the event with the highest `created_at` per `(kind, pubkey, d_tag)`. + /// Same-second ties are broken by lowest event `id` (deterministic ordering). + /// The entire check → soft-delete → insert runs in a single transaction with + /// an advisory lock to prevent concurrent-insert races. + /// + /// **Channel policy:** NIP-33 replacement keys on `(kind, pubkey, d_tag)` globally — + /// `channel_id` is NOT part of the replacement key. This matches the Nostr spec: + /// an author's parameterized replaceable event is a single global resource identified + /// by its d-tag, regardless of which channel it was submitted to. The `channel_id` + /// parameter is stored on the new row for query scoping but does not affect replacement. + /// + /// Note: `replace_addressable_event()` keys on `channel_id` because it serves + /// relay-signed NIP-29 group metadata (kind 39000–39002) where the relay is the + /// author and channel_id distinguishes groups. User-submitted NIP-33 events use + /// this function instead, where the author's pubkey + d-tag is the natural key. + pub async fn replace_parameterized_event( + &self, + event: &nostr::Event, + d_tag: &str, + channel_id: Option, + ) -> Result<(StoredEvent, bool)> { + let kind_i32 = sprout_core::kind::event_kind_i32(event); + let pubkey_bytes = event.pubkey.to_bytes(); + let created_at_secs = event.created_at.as_u64() as i64; + let created_at = chrono::DateTime::from_timestamp(created_at_secs, 0) + .ok_or(DbError::InvalidTimestamp(created_at_secs))?; + + // Stable advisory-lock key: FNV-1a over (kind, pubkey, d_tag). + // Same algorithm as replace_addressable_event — deterministic across processes. + let lock_key = { + let mut h: u64 = 0xcbf29ce484222325; // FNV offset basis + for b in kind_i32.to_le_bytes() { + h ^= b as u64; + h = h.wrapping_mul(0x100000001b3); + } + for b in pubkey_bytes.as_slice() { + h ^= *b as u64; + h = h.wrapping_mul(0x100000001b3); + } + for b in d_tag.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x100000001b3); + } + h as i64 + }; + + let mut tx = self.pool.begin().await?; + + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(lock_key) + .execute(&mut *tx) + .await?; + + // Check for existing event with same (kind, pubkey, d_tag). + let existing: Option<(chrono::DateTime, Vec)> = sqlx::query_as( + "SELECT created_at, id FROM events \ + WHERE kind = $1 AND pubkey = $2 AND d_tag = $3 AND deleted_at IS NULL \ + ORDER BY created_at DESC, id ASC LIMIT 1", + ) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(d_tag) + .fetch_optional(&mut *tx) + .await?; + + // Stale-write protection: reject if incoming is not newer. + let incoming_id = event.id.as_bytes().as_slice(); + if let Some((existing_ts, existing_id)) = existing { + let dominated = created_at < existing_ts + || (created_at == existing_ts && incoming_id >= existing_id.as_slice()); + if dominated { + tx.rollback().await?; + let received_at = chrono::Utc::now(); + return Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, false), + false, + )); + } + + // Soft-delete the older event(s). + sqlx::query( + "UPDATE events SET deleted_at = NOW() \ + WHERE kind = $1 AND pubkey = $2 AND d_tag = $3 AND deleted_at IS NULL", + ) + .bind(kind_i32) + .bind(pubkey_bytes.as_slice()) + .bind(d_tag) + .execute(&mut *tx) + .await?; + } + + // Insert the new event inside the transaction. + let sig_bytes = event.sig.serialize(); + let tags_json = serde_json::to_value(&event.tags)?; + let received_at = chrono::Utc::now(); + + let insert_result = sqlx::query( + "INSERT INTO events (id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id, d_tag) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ + ON CONFLICT DO NOTHING", + ) + .bind(event.id.as_bytes().as_slice()) + .bind(pubkey_bytes.as_slice()) + .bind(created_at) + .bind(kind_i32) + .bind(&tags_json) + .bind(&event.content) + .bind(sig_bytes.as_slice()) + .bind(received_at) + .bind(channel_id) + .bind(d_tag) + .execute(&mut *tx) + .await?; + + let was_inserted = insert_result.rows_affected() > 0; + if !was_inserted { + tx.rollback().await?; + return Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, false), + false, + )); + } + + tx.commit().await?; + + // Mentions are a denormalized index — safe outside the transaction. + if let Err(e) = crate::insert_mentions(&self.pool, event, channel_id).await { + tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); + } + + Ok(( + StoredEvent::with_received_at(event.clone(), received_at, channel_id, true), + true, + )) + } } /// A full API token record. diff --git a/crates/sprout-relay/src/handlers/ingest.rs b/crates/sprout-relay/src/handlers/ingest.rs index c5e0ce9d..6b00a798 100644 --- a/crates/sprout-relay/src/handlers/ingest.rs +++ b/crates/sprout-relay/src/handlers/ingest.rs @@ -12,14 +12,15 @@ use uuid::Uuid; use nostr::Event; use sprout_auth::Scope; use sprout_core::kind::{ - event_kind_u32, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, KIND_DELETION, KIND_FORUM_COMMENT, - KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, KIND_MEMBER_ADDED_NOTIFICATION, - KIND_MEMBER_REMOVED_NOTIFICATION, KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, - KIND_NIP29_DELETE_GROUP, KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, - KIND_NIP29_LEAVE_REQUEST, KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, - KIND_PROFILE, KIND_REACTION, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, - KIND_STREAM_MESSAGE_DIFF, KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, - KIND_STREAM_MESSAGE_SCHEDULED, KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, + event_kind_u32, is_parameterized_replaceable, KIND_AUTH, KIND_CANVAS, KIND_CONTACT_LIST, + KIND_DELETION, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_GIFT_WRAP, + KIND_LONG_FORM, KIND_MEMBER_ADDED_NOTIFICATION, KIND_MEMBER_REMOVED_NOTIFICATION, + KIND_NIP29_CREATE_GROUP, KIND_NIP29_DELETE_EVENT, KIND_NIP29_DELETE_GROUP, + KIND_NIP29_EDIT_METADATA, KIND_NIP29_JOIN_REQUEST, KIND_NIP29_LEAVE_REQUEST, + KIND_NIP29_PUT_USER, KIND_NIP29_REMOVE_USER, KIND_PRESENCE_UPDATE, KIND_PROFILE, KIND_REACTION, + KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_BOOKMARKED, KIND_STREAM_MESSAGE_DIFF, + KIND_STREAM_MESSAGE_EDIT, KIND_STREAM_MESSAGE_PINNED, KIND_STREAM_MESSAGE_SCHEDULED, + KIND_STREAM_MESSAGE_V2, KIND_STREAM_REMINDER, KIND_TEXT_NOTE, }; use sprout_core::verification::verify_event; @@ -141,7 +142,7 @@ pub enum IngestError { fn required_scope_for_kind(kind: u32, event: &Event) -> Result { match kind { KIND_PROFILE => Ok(Scope::UsersWrite), - KIND_TEXT_NOTE => Ok(Scope::MessagesWrite), + KIND_TEXT_NOTE | KIND_LONG_FORM => Ok(Scope::MessagesWrite), KIND_CONTACT_LIST => Ok(Scope::UsersWrite), KIND_DELETION | KIND_REACTION @@ -241,6 +242,24 @@ pub(crate) async fn derive_reaction_channel( } } +/// Kinds that are always global (`channel_id = NULL`). +/// +/// If a client includes a stray `h` tag on these kinds, the ingest pipeline +/// sets `channel_id = None` — these events are never channel-scoped. +/// +/// Note: the raw `h` tag remains on the stored event (Nostr events are signed, +/// so tags cannot be stripped without invalidating the signature). The read-path +/// filter matching in `filter.rs` treats explicit `h` tags as authoritative, +/// which means a stray `h` tag can still match `#h` queries. This is a known +/// limitation affecting all global-only kinds and should be addressed in the +/// filter layer as a follow-up. +pub(crate) fn is_global_only_kind(kind: u32) -> bool { + matches!( + kind, + KIND_PROFILE | KIND_TEXT_NOTE | KIND_CONTACT_LIST | KIND_LONG_FORM + ) +} + /// Kinds that require an `h` tag for channel scoping. pub(crate) fn requires_h_channel_scope(kind: u32) -> bool { matches!( @@ -843,9 +862,7 @@ pub async fn ingest_event( }; // ── 5b. Global-only kinds ignore h-tags ───────────────────────────── - // kind:0 (profile), kind:1 (text note), kind:3 (contact list) are always global. - // If a client includes a stray h-tag, ignore it — these kinds are never channel-scoped. - if matches!(kind_u32, KIND_PROFILE | KIND_TEXT_NOTE | KIND_CONTACT_LIST) { + if is_global_only_kind(kind_u32) { channel_id = None; } @@ -1238,6 +1255,21 @@ pub async fn ingest_event( .replace_addressable_event(&event, channel_id) .await .map_err(|e| IngestError::Internal(format!("error: {e}")))? + } else if is_parameterized_replaceable(kind_u32) { + // NIP-33 parameterized replaceable — keyed by (kind, pubkey, d_tag). + let d_tag = sprout_db::event::extract_d_tag(&event).unwrap_or_default(); + if d_tag.len() > sprout_db::event::D_TAG_MAX_LEN { + return Err(IngestError::Rejected(format!( + "invalid: d tag too long ({} bytes, max {})", + d_tag.len(), + sprout_db::event::D_TAG_MAX_LEN, + ))); + } + state + .db + .replace_parameterized_event(&event, &d_tag, channel_id) + .await + .map_err(|e| IngestError::Internal(format!("error: {e}")))? } else { let thread_params = thread_meta.as_ref().map(|m| m.as_params()); match state @@ -1298,8 +1330,8 @@ pub async fn ingest_event( mod tests { use super::*; use sprout_core::kind::{ - KIND_CANVAS, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_PRESENCE_UPDATE, - KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_DIFF, + KIND_CANVAS, KIND_FORUM_COMMENT, KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_LONG_FORM, + KIND_PRESENCE_UPDATE, KIND_STREAM_MESSAGE, KIND_STREAM_MESSAGE_DIFF, }; #[test] @@ -1355,13 +1387,54 @@ mod tests { assert!(!requires_h_channel_scope(KIND_REACTION)); } + #[test] + fn long_form_is_in_scope_allowlist() { + let dummy = make_dummy_event(); + assert!( + required_scope_for_kind(KIND_LONG_FORM, &dummy).is_ok(), + "KIND_LONG_FORM (30023) should be accepted" + ); + } + + #[test] + fn long_form_requires_messages_write_scope() { + let dummy = make_dummy_event(); + assert_eq!( + required_scope_for_kind(KIND_LONG_FORM, &dummy).unwrap(), + Scope::MessagesWrite, + ); + } + + #[test] + fn long_form_does_not_require_h_tag() { + // kind:30023 is global (author-owned, not channel-scoped) + assert!(!requires_h_channel_scope(KIND_LONG_FORM)); + } + + #[test] + fn long_form_is_global_only() { + // kind:30023 is always global — ingest nulls channel_id even if an h-tag is present + assert!(is_global_only_kind(KIND_LONG_FORM)); + } + + #[test] + fn global_only_and_channel_scoped_are_disjoint() { + // A kind cannot be both global-only and channel-scoped + for kind in 0..=65535u32 { + assert!( + !(is_global_only_kind(kind) && requires_h_channel_scope(kind)), + "kind {kind} is both global-only and channel-scoped" + ); + } + } + #[test] fn ephemeral_kinds_not_in_scope_allowlist() { assert!(required_scope_for_kind(KIND_PRESENCE_UPDATE, &make_dummy_event()).is_err()); } #[test] - fn per_kind_scope_allowlist_covers_all_18_migrated_kinds() { + fn per_kind_scope_allowlist_covers_all_migrated_kinds() { let dummy = make_dummy_event(); let migrated = [ KIND_PROFILE, @@ -1382,6 +1455,7 @@ mod tests { KIND_FORUM_POST, KIND_FORUM_VOTE, KIND_FORUM_COMMENT, + KIND_LONG_FORM, ]; for kind in migrated { assert!( diff --git a/crates/sprout-relay/src/handlers/req.rs b/crates/sprout-relay/src/handlers/req.rs index 8dc6e61d..6cdf21b9 100644 --- a/crates/sprout-relay/src/handlers/req.rs +++ b/crates/sprout-relay/src/handlers/req.rs @@ -469,6 +469,33 @@ fn filter_to_query_params(filter: &Filter, channel_id: Option) -> Ev } }); + // Push single-value #d tag into SQL via the d_tag column (NIP-33). + // Critical for parameterized replaceable lookups (authors + kinds + #d) + // where many events from the same author would push the target past LIMIT. + // + // Only push when the filter exclusively targets NIP-33 kinds (30000–39999), + // because `d_tag` is only populated for those kinds. Non-NIP-33 events have + // `d_tag = NULL`, so pushing `AND d_tag = $N` for a mixed-kind or kindless + // filter would silently exclude non-NIP-33 rows that match via their tags. + let filter_is_nip33_only = kinds.as_ref().is_some_and(|ks| { + !ks.is_empty() + && ks + .iter() + .all(|&k| sprout_core::kind::is_parameterized_replaceable(k as u32)) + }); + let d_tag_key = nostr::SingleLetterTag::lowercase(nostr::Alphabet::D); + let d_tag = if filter_is_nip33_only { + filter.generic_tags.get(&d_tag_key).and_then(|values| { + if values.len() == 1 { + values.iter().next().map(|v| v.to_string()) + } else { + None + } + }) + } else { + None + }; + EventQuery { channel_id, kinds, @@ -477,6 +504,7 @@ fn filter_to_query_params(filter: &Filter, channel_id: Option) -> Ev until, limit: Some(limit), p_tag_hex, + d_tag, ..Default::default() } } @@ -601,4 +629,42 @@ mod tests { assert!(has_search); assert!(!has_non_search, "all-search filters should not be mixed"); } + + #[test] + fn d_tag_pushdown_only_for_nip33_kinds() { + let d_tag = SingleLetterTag::lowercase(Alphabet::D); + + // NIP-33 kind with #d → pushdown active + let nip33_filter = Filter::new() + .kind(nostr::Kind::Custom(30023)) + .custom_tag(d_tag, ["my-slug"]); + let q = filter_to_query_params(&nip33_filter, None); + assert_eq!(q.d_tag, Some("my-slug".to_string())); + + // Non-NIP-33 kind with #d → pushdown NOT active (would miss rows with d_tag=NULL) + let non_nip33_filter = Filter::new() + .kind(nostr::Kind::Custom(1)) + .custom_tag(d_tag, ["some-value"]); + let q2 = filter_to_query_params(&non_nip33_filter, None); + assert_eq!(q2.d_tag, None); + + // Mixed kinds (one NIP-33, one not) → pushdown NOT active + let mixed_filter = Filter::new() + .kinds([nostr::Kind::Custom(30023), nostr::Kind::Custom(1)]) + .custom_tag(d_tag, ["slug"]); + let q3 = filter_to_query_params(&mixed_filter, None); + assert_eq!(q3.d_tag, None); + + // No kinds specified → pushdown NOT active + let no_kinds_filter = Filter::new().custom_tag(d_tag, ["slug"]); + let q4 = filter_to_query_params(&no_kinds_filter, None); + assert_eq!(q4.d_tag, None); + + // Multi-value #d → pushdown NOT active (can't push OR into single column match) + let multi_d_filter = Filter::new() + .kind(nostr::Kind::Custom(30023)) + .custom_tag(d_tag, ["slug-a", "slug-b"]); + let q5 = filter_to_query_params(&multi_d_filter, None); + assert_eq!(q5.d_tag, None); + } } diff --git a/crates/sprout-relay/src/main.rs b/crates/sprout-relay/src/main.rs index 8a9af769..4f9c8489 100644 --- a/crates/sprout-relay/src/main.rs +++ b/crates/sprout-relay/src/main.rs @@ -59,6 +59,14 @@ async fn main() -> anyhow::Result<()> { error!("Failed to ensure partitions: {e}"); } + // NIP-33: backfill d_tag for any existing parameterized replaceable events + // that predate the column addition. Idempotent — no-ops when fully populated. + match db.backfill_d_tags().await { + Ok(0) => {} + Ok(n) => info!("Backfilled d_tag for {n} NIP-33 events"), + Err(e) => error!("Failed to backfill d_tags: {e}"), + } + let audit_pool = sqlx::PgPool::connect(&config.database_url) .await .map_err(|e| anyhow::anyhow!("Audit DB connection failed: {e}"))?; diff --git a/crates/sprout-relay/src/nip11.rs b/crates/sprout-relay/src/nip11.rs index f7ab0e9e..c9cf2e62 100644 --- a/crates/sprout-relay/src/nip11.rs +++ b/crates/sprout-relay/src/nip11.rs @@ -4,6 +4,11 @@ use serde::{Deserialize, Serialize}; use crate::connection::MAX_FRAME_BYTES; +/// NIPs supported by this relay, advertised in the NIP-11 document. +/// Kept as a module-level constant so tests can verify it without constructing +/// a full `Config` (which reads env vars and races with config.rs tests). +pub(crate) const SUPPORTED_NIPS: &[u32] = &[1, 2, 10, 11, 16, 17, 23, 25, 29, 33, 42, 50]; + /// Relay information document served at `GET /` with `Accept: application/nostr+json`. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RelayInfo { @@ -56,7 +61,7 @@ impl RelayInfo { description: "Sprout — private team communication relay".to_string(), pubkey: None, contact: None, - supported_nips: vec![1, 2, 10, 11, 16, 17, 25, 29, 42, 50], + supported_nips: SUPPORTED_NIPS.to_vec(), software: "https://github.com/sprout-rs/sprout".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), limitation: Some(RelayLimitation { @@ -80,3 +85,33 @@ pub async fn relay_info_handler( ) -> axum::response::Json { axum::response::Json(RelayInfo::from_config(&state.config)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supported_nips_includes_nip23_and_nip33() { + // Tests the production SUPPORTED_NIPS constant directly — no Config::from_env() + // needed, avoiding the env-var race with config.rs tests. + assert!( + SUPPORTED_NIPS.contains(&23), + "NIP-23 (long-form content) must be advertised" + ); + assert!( + SUPPORTED_NIPS.contains(&33), + "NIP-33 (parameterized replaceable) must be advertised" + ); + } + + #[test] + fn supported_nips_are_sorted() { + let mut sorted = SUPPORTED_NIPS.to_vec(); + sorted.sort(); + assert_eq!( + SUPPORTED_NIPS, + &sorted[..], + "supported_nips should be sorted" + ); + } +} diff --git a/crates/sprout-test-client/tests/e2e_long_form.rs b/crates/sprout-test-client/tests/e2e_long_form.rs new file mode 100644 index 00000000..ff2c4934 --- /dev/null +++ b/crates/sprout-test-client/tests/e2e_long_form.rs @@ -0,0 +1,319 @@ +//! End-to-end tests for NIP-23 long-form content (kind:30023). +//! +//! These tests require a running relay instance. By default they are marked +//! `#[ignore]` so that `cargo test` does not fail in CI when the relay is not +//! available. +//! +//! # Running +//! +//! Start the relay, then run: +//! +//! ```text +//! cargo test --test e2e_long_form -- --ignored +//! ``` +//! +//! Override the relay URL with the `RELAY_URL` environment variable: +//! +//! ```text +//! RELAY_URL=ws://relay.example.com cargo test --test e2e_long_form -- --ignored +//! ``` + +use std::time::Duration; + +use nostr::{Alphabet, EventBuilder, Filter, Keys, Kind, SingleLetterTag, Tag, Timestamp}; +use sprout_test_client::SproutTestClient; + +const KIND_LONG_FORM: u16 = 30023; + +fn relay_url() -> String { + std::env::var("RELAY_URL").unwrap_or_else(|_| "ws://localhost:3000".to_string()) +} + +fn sub_id(name: &str) -> String { + format!("e2e-{name}-{}", uuid::Uuid::new_v4()) +} + +/// Build a kind:30023 event with standard NIP-23 tags. +fn build_long_form_event( + keys: &Keys, + d_tag: &str, + title: &str, + content: &str, + extra_tags: Vec, +) -> nostr::Event { + let mut tags = vec![ + Tag::parse(&["d", d_tag]).unwrap(), + Tag::parse(&["title", title]).unwrap(), + ]; + tags.extend(extra_tags); + EventBuilder::new(Kind::Custom(KIND_LONG_FORM), content, tags) + .sign_with_keys(keys) + .unwrap() +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +/// kind:30023 events are accepted by the relay. +#[tokio::test] +#[ignore] +async fn test_long_form_accepted() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + let event = build_long_form_event( + &keys, + "test-article-accept", + "Test Article", + "# Hello\n\nThis is a test article.", + vec![], + ); + + let ok = client.send_event(event).await.expect("send event"); + assert!( + ok.accepted, + "relay should accept kind:30023: {}", + ok.message + ); + + client.disconnect().await.expect("disconnect"); +} + +/// kind:30023 events are retrievable via REQ with kinds filter. +#[tokio::test] +#[ignore] +async fn test_long_form_retrievable() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + let d_tag = format!("retrieve-{}", uuid::Uuid::new_v4().simple()); + let event = build_long_form_event( + &keys, + &d_tag, + "Retrievable Article", + "# Retrievable\n\nBody text.", + vec![], + ); + let event_id = event.id; + + let ok = client.send_event(event).await.expect("send event"); + assert!(ok.accepted, "relay should accept: {}", ok.message); + + // Query back by kind + author + let sid = sub_id("retrieve"); + let filter = Filter::new() + .kind(Kind::Custom(KIND_LONG_FORM)) + .author(keys.public_key()); + client + .subscribe(&sid, vec![filter]) + .await + .expect("subscribe"); + + let events = client + .collect_until_eose(&sid, Duration::from_secs(5)) + .await + .expect("collect"); + + assert!( + events.iter().any(|e| e.id == event_id), + "should find the published article in query results" + ); + + client.disconnect().await.expect("disconnect"); +} + +/// kind:30023 is stored globally (channel_id = NULL) — stray h-tags are ignored. +/// An event with a stray h-tag should still be retrievable via a global query +/// (no h-tag filter), proving it was stored as global. +#[tokio::test] +#[ignore] +async fn test_long_form_stray_h_tag_ignored() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + // Publish with a stray h-tag (a UUID that doesn't correspond to any channel). + let fake_channel = uuid::Uuid::new_v4().to_string(); + let d_tag = format!("stray-h-{}", uuid::Uuid::new_v4().simple()); + let event = build_long_form_event( + &keys, + &d_tag, + "Stray H-Tag Article", + "Should be stored globally despite h-tag.", + vec![Tag::parse(&["h", &fake_channel]).unwrap()], + ); + let event_id = event.id; + + let ok = client.send_event(event).await.expect("send event"); + assert!(ok.accepted, "relay should accept: {}", ok.message); + + // Query globally (no h-tag filter) — should find the article. + let sid = sub_id("stray-h"); + let filter = Filter::new() + .kind(Kind::Custom(KIND_LONG_FORM)) + .author(keys.public_key()); + client + .subscribe(&sid, vec![filter]) + .await + .expect("subscribe"); + + let events = client + .collect_until_eose(&sid, Duration::from_secs(5)) + .await + .expect("collect"); + + assert!( + events.iter().any(|e| e.id == event_id), + "article with stray h-tag should be retrievable via global query" + ); + + // NOTE: Ideally, querying with #h= should NOT return the + // article since it's global. However, the raw h-tag remains on the stored + // event (Nostr events are signed — tags can't be stripped without breaking + // the signature), and the read-path filter matching in filter.rs treats + // explicit h-tags as authoritative. This is a pre-existing limitation + // affecting all global-only kinds (0, 1, 3, 30023) and should be fixed + // in the filter layer as a follow-up. + + client.disconnect().await.expect("disconnect"); +} + +/// NIP-33 replacement: publishing a newer kind:30023 with the same d-tag replaces the old one. +#[tokio::test] +#[ignore] +async fn test_long_form_nip33_replacement() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + let d_tag = format!("replace-{}", uuid::Uuid::new_v4().simple()); + + // Publish v1 + let v1 = build_long_form_event(&keys, &d_tag, "Article v1", "Version 1 content.", vec![]); + let ok1 = client.send_event(v1).await.expect("send v1"); + assert!(ok1.accepted, "v1 should be accepted: {}", ok1.message); + + // Small delay to ensure different created_at timestamps + tokio::time::sleep(Duration::from_secs(1)).await; + + // Publish v2 with the same d-tag + let v2 = build_long_form_event( + &keys, + &d_tag, + "Article v2", + "Version 2 content — updated.", + vec![], + ); + let v2_id = v2.id; + let ok2 = client.send_event(v2).await.expect("send v2"); + assert!(ok2.accepted, "v2 should be accepted: {}", ok2.message); + + // Query — should only get v2 (v1 replaced) + let sid = sub_id("replace"); + let filter = Filter::new() + .kind(Kind::Custom(KIND_LONG_FORM)) + .author(keys.public_key()) + .custom_tag(SingleLetterTag::lowercase(Alphabet::D), [d_tag.as_str()]); + client + .subscribe(&sid, vec![filter]) + .await + .expect("subscribe"); + + let events = client + .collect_until_eose(&sid, Duration::from_secs(5)) + .await + .expect("collect"); + + assert_eq!( + events.len(), + 1, + "should have exactly one event after replacement" + ); + assert_eq!(events[0].id, v2_id, "surviving event should be v2"); + assert!( + events[0].content.contains("Version 2"), + "content should be v2" + ); + + client.disconnect().await.expect("disconnect"); +} + +/// NIP-33 stale-write protection: an older event cannot replace a newer one. +#[tokio::test] +#[ignore] +async fn test_long_form_stale_write_rejected() { + let url = relay_url(); + let keys = Keys::generate(); + let mut client = SproutTestClient::connect(&url, &keys) + .await + .expect("connect"); + + let d_tag = format!("stale-{}", uuid::Uuid::new_v4().simple()); + + // Publish the "newer" event first (with a future-ish timestamp) + let newer = { + let tags = vec![ + Tag::parse(&["d", &d_tag]).unwrap(), + Tag::parse(&["title", "Newer Article"]).unwrap(), + ]; + EventBuilder::new(Kind::Custom(KIND_LONG_FORM), "Newer content.", tags) + .custom_created_at(Timestamp::from(nostr::Timestamp::now().as_u64() + 100)) + .sign_with_keys(&keys) + .unwrap() + }; + let newer_id = newer.id; + let ok1 = client.send_event(newer).await.expect("send newer"); + assert!(ok1.accepted, "newer should be accepted: {}", ok1.message); + + // Now try to publish an "older" event with the same d-tag but earlier timestamp + let older = { + let tags = vec![ + Tag::parse(&["d", &d_tag]).unwrap(), + Tag::parse(&["title", "Older Article"]).unwrap(), + ]; + EventBuilder::new(Kind::Custom(KIND_LONG_FORM), "Older content.", tags) + .custom_created_at(Timestamp::from(nostr::Timestamp::now().as_u64() - 100)) + .sign_with_keys(&keys) + .unwrap() + }; + let _ok2 = client.send_event(older).await.expect("send older"); + // Stale write may be rejected or accepted-as-duplicate — either way, + // the older event must NOT replace the newer one. + + // Query — should still have the newer event + let sid = sub_id("stale"); + let filter = Filter::new() + .kind(Kind::Custom(KIND_LONG_FORM)) + .author(keys.public_key()) + .custom_tag(SingleLetterTag::lowercase(Alphabet::D), [d_tag.as_str()]); + client + .subscribe(&sid, vec![filter]) + .await + .expect("subscribe"); + + let events = client + .collect_until_eose(&sid, Duration::from_secs(5)) + .await + .expect("collect"); + + assert_eq!(events.len(), 1, "should have exactly one event"); + assert_eq!( + events[0].id, newer_id, + "surviving event should be the newer one" + ); + assert!( + events[0].content.contains("Newer"), + "content should be from the newer event" + ); + + client.disconnect().await.expect("disconnect"); +} diff --git a/schema/schema.sql b/schema/schema.sql index e5e82ae6..01c11f11 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -94,6 +94,7 @@ CREATE TABLE events ( received_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), channel_id UUID, deleted_at TIMESTAMPTZ, + d_tag TEXT, PRIMARY KEY (created_at, id) ) PARTITION BY RANGE (created_at); @@ -120,6 +121,7 @@ CREATE INDEX idx_events_kind_created ON events (kind, created_at); CREATE INDEX idx_events_id ON events (id); CREATE INDEX idx_events_deleted ON events (deleted_at); CREATE INDEX idx_events_addressable ON events (kind, pubkey, channel_id, deleted_at); +CREATE INDEX idx_events_parameterized ON events (kind, pubkey, d_tag, deleted_at) WHERE d_tag IS NOT NULL; -- ── Event mentions ──────────────────────────────────────────────────────────── diff --git a/scripts/backfill-d-tag.sql b/scripts/backfill-d-tag.sql new file mode 100644 index 00000000..8684accb --- /dev/null +++ b/scripts/backfill-d-tag.sql @@ -0,0 +1,17 @@ +-- Backfill d_tag for existing NIP-33 range events (kind 30000–39999). +-- Idempotent: only updates rows where d_tag is still NULL. +-- Includes soft-deleted rows so the column is fully populated. +-- Run once after adding the d_tag column to the events table. +-- +-- Usage: psql $DATABASE_URL -f scripts/backfill-d-tag.sql + +UPDATE events +SET d_tag = COALESCE( + (SELECT elem->>1 + FROM jsonb_array_elements(tags) AS elem + WHERE elem->>0 = 'd' + LIMIT 1), + '' +) +WHERE kind BETWEEN 30000 AND 39999 + AND d_tag IS NULL; diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh index f7991cfe..005308ba 100755 --- a/scripts/dev-setup.sh +++ b/scripts/dev-setup.sh @@ -151,6 +151,17 @@ else sleep 2 done success "Migrations applied via pgschema" + + # Run data backfills (idempotent — safe to re-run). + BACKFILL_DIR="${REPO_ROOT}/scripts" + if [[ -f "${BACKFILL_DIR}/backfill-d-tag.sql" ]]; then + log "Running d_tag backfill for NIP-33 events..." + if psql "${DATABASE_URL}" -f "${BACKFILL_DIR}/backfill-d-tag.sql" 2>/dev/null; then + success "d_tag backfill complete" + else + warn "d_tag backfill failed (relay startup will retry automatically)" + fi + fi else error "pgschema not found at ${PGSCHEMA}. Run: ./bin/hermit install pgschema" exit 1