diff --git a/CLI.md b/CLI.md index 64cc4e6665a4..9a8c0779c358 100644 --- a/CLI.md +++ b/CLI.md @@ -218,6 +218,9 @@ Client implementation and command-line tool for the Linera blockchain * `--sender-certificate-download-batch-size ` — Maximum number of sender certificates we try to download and receive in one go when syncing sender chains Default value: `20000` +* `--received-log-pages-before-pacing ` — Number of full received-log pages to fetch back-to-back before pausing between pages. Serving a page makes the validator read one storage row per log entry, so an unthrottled walk of a long backlog degrades its read latency for every other client. Set to 0 to pace from the first page + + Default value: `5` * `--max-concurrent-batch-downloads ` — Maximum number of certificate batches downloaded concurrently during chain sync Default value: `1` diff --git a/linera-client/src/client_options.rs b/linera-client/src/client_options.rs index d65135bcb77a..c11b0d3fbb84 100644 --- a/linera-client/src/client_options.rs +++ b/linera-client/src/client_options.rs @@ -16,7 +16,8 @@ use linera_core::{ client::{ chain_client, DEFAULT_CERTIFICATE_DOWNLOAD_BATCH_SIZE, DEFAULT_CERTIFICATE_UPLOAD_BATCH_SIZE, DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS, - DEFAULT_MAX_EVENT_STREAM_QUERIES, DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE, + DEFAULT_MAX_EVENT_STREAM_QUERIES, DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING, + DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE, }, node::CrossChainMessageDelivery, DEFAULT_QUORUM_GRACE_PERIOD, @@ -247,6 +248,13 @@ pub struct Options { )] pub sender_certificate_download_batch_size: usize, + /// Number of full received-log pages to fetch back-to-back before pausing between pages. + /// Serving a page makes the validator read one storage row per log entry, so an + /// unthrottled walk of a long backlog degrades its read latency for every other client. + /// Set to 0 to pace from the first page. + #[arg(long, default_value_t = DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING)] + pub received_log_pages_before_pacing: usize, + /// Maximum number of certificate batches downloaded concurrently during chain sync. #[arg(long, default_value_t = DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS)] pub max_concurrent_batch_downloads: usize, @@ -369,6 +377,7 @@ impl Options { certificate_download_batch_size: self.certificate_download_batch_size, certificate_upload_batch_size: self.certificate_upload_batch_size, sender_certificate_download_batch_size: self.sender_certificate_download_batch_size, + received_log_pages_before_pacing: self.received_log_pages_before_pacing, max_concurrent_batch_downloads: self.max_concurrent_batch_downloads, max_joined_tasks: self.max_joined_tasks, allow_fast_blocks: self.allow_fast_blocks, diff --git a/linera-core/src/client/chain_client/mod.rs b/linera-core/src/client/chain_client/mod.rs index 7c9d4ce0ba28..7f81cf53cd2b 100644 --- a/linera-core/src/client/chain_client/mod.rs +++ b/linera-core/src/client/chain_client/mod.rs @@ -132,6 +132,9 @@ pub struct Options { /// Maximum number of sender certificates we try to download and receive in one go /// when syncing sender chains. pub sender_certificate_download_batch_size: usize, + /// Number of full received-log pages a download may fetch back-to-back before it starts + /// pausing between pages. Set to 0 to pace from the first page. + pub received_log_pages_before_pacing: usize, /// Maximum number of certificate batches downloaded concurrently during chain sync. pub max_concurrent_batch_downloads: usize, /// Maximum number of tasks that can be joined concurrently using buffer_unordered. @@ -262,6 +265,7 @@ impl Options { use super::{ DEFAULT_CERTIFICATE_DOWNLOAD_BATCH_SIZE, DEFAULT_CERTIFICATE_UPLOAD_BATCH_SIZE, DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS, DEFAULT_MAX_EVENT_STREAM_QUERIES, + DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING, DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE, }; use crate::DEFAULT_QUORUM_GRACE_PERIOD; @@ -279,6 +283,7 @@ impl Options { certificate_download_batch_size: DEFAULT_CERTIFICATE_DOWNLOAD_BATCH_SIZE, certificate_upload_batch_size: DEFAULT_CERTIFICATE_UPLOAD_BATCH_SIZE, sender_certificate_download_batch_size: DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE, + received_log_pages_before_pacing: DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING, max_concurrent_batch_downloads: DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS, max_joined_tasks: 100, allow_fast_blocks: false, diff --git a/linera-core/src/client/mod.rs b/linera-core/src/client/mod.rs index faccdb400a52..e51ac927744a 100644 --- a/linera-core/src/client/mod.rs +++ b/linera-core/src/client/mod.rs @@ -177,6 +177,10 @@ pub static DEFAULT_CERTIFICATE_UPLOAD_BATCH_SIZE: usize = 500; pub static DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE: usize = 20_000; /// Default maximum number of concurrent event stream queries. pub static DEFAULT_MAX_EVENT_STREAM_QUERIES: usize = 1000; +/// Default number of full received-log pages a download may fetch back-to-back before pacing +/// kicks in. With pages of [`CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES`] entries this lets backlogs +/// of up to just under 100,000 entries sync at full speed. +pub static DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING: usize = 5; /// Default maximum number of certificate batch downloads to run concurrently. pub static DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS: usize = 1; @@ -1884,10 +1888,13 @@ impl Client { // Retrieve the list of newly received certificates from this validator. let mut remote_log = Vec::new(); let mut num_pages = 0usize; + let mut num_full_pages = 0usize; loop { trace!("get_received_log_from_validator: looping"); let query = ChainInfoQuery::new(chain_id).with_received_log_excluding_first_n(offset); + let page_start = linera_base::time::Instant::now(); let info = remote_node.handle_chain_info_query(query).await?; + let page_duration = page_start.elapsed(); let received_entries = info.requested_received_log.len(); offset += received_entries as u64; remote_log.extend(info.requested_received_log); @@ -1908,6 +1915,17 @@ impl Client { if received_entries < CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES { break; } + // Serving a page makes the validator read one storage row per log entry, so + // walking a long backlog page after page can monopolize the validator's + // storage and degrade its read latency for everyone else. Once the walk is + // clearly a backlog walk, pause between pages for as long as the previous + // page took: this caps the walk at roughly half of the validator's serving + // capacity and automatically backs off further when the validator slows + // down. Short syncs never reach this point and are unaffected. + num_full_pages += 1; + if num_full_pages >= self.options.received_log_pages_before_pacing { + linera_base::time::timer::sleep(page_duration).await; + } } trace!(