Pace long received-log downloads to avoid monopolizing validator storage

CLI.md

@@ -218,6 +218,9 @@ Client implementation and command-line tool for the Linera blockchain
 * `--sender-certificate-download-batch-size <SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE>` — Maximum number of sender certificates we try to download and receive in one go when syncing sender chains
 
   Default value: `20000`
+* `--received-log-pages-before-pacing <RECEIVED_LOG_PAGES_BEFORE_PACING>` — Number of full received-log pages to fetch back-to-back before pausing between pages. Serving a page makes the validator read one storage row per log entry, so an unthrottled walk of a long backlog degrades its read latency for every other client. Set to 0 to pace from the first page
+
+  Default value: `5`
 * `--max-concurrent-batch-downloads <MAX_CONCURRENT_BATCH_DOWNLOADS>` — Maximum number of certificate batches downloaded concurrently during chain sync
 
   Default value: `1`

linera-client/src/client_options.rs

@@ -16,7 +16,8 @@ use linera_core::{
     client::{
         chain_client, DEFAULT_CERTIFICATE_DOWNLOAD_BATCH_SIZE,
         DEFAULT_CERTIFICATE_UPLOAD_BATCH_SIZE, DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS,
-        DEFAULT_MAX_EVENT_STREAM_QUERIES, DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE,
+        DEFAULT_MAX_EVENT_STREAM_QUERIES, DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING,
+        DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE,
     },
     node::CrossChainMessageDelivery,
     DEFAULT_QUORUM_GRACE_PERIOD,
@@ -247,6 +248,13 @@ pub struct Options {
     )]
     pub sender_certificate_download_batch_size: usize,
 
+    /// Number of full received-log pages to fetch back-to-back before pausing between pages.
+    /// Serving a page makes the validator read one storage row per log entry, so an
+    /// unthrottled walk of a long backlog degrades its read latency for every other client.
+    /// Set to 0 to pace from the first page.
+    #[arg(long, default_value_t = DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING)]
+    pub received_log_pages_before_pacing: usize,
+
     /// Maximum number of certificate batches downloaded concurrently during chain sync.
     #[arg(long, default_value_t = DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS)]
     pub max_concurrent_batch_downloads: usize,
@@ -369,6 +377,7 @@ impl Options {
             certificate_download_batch_size: self.certificate_download_batch_size,
             certificate_upload_batch_size: self.certificate_upload_batch_size,
             sender_certificate_download_batch_size: self.sender_certificate_download_batch_size,
+            received_log_pages_before_pacing: self.received_log_pages_before_pacing,
             max_concurrent_batch_downloads: self.max_concurrent_batch_downloads,
             max_joined_tasks: self.max_joined_tasks,
             allow_fast_blocks: self.allow_fast_blocks,

linera-core/src/client/chain_client/mod.rs

@@ -132,6 +132,9 @@ pub struct Options {
     /// Maximum number of sender certificates we try to download and receive in one go
     /// when syncing sender chains.
     pub sender_certificate_download_batch_size: usize,
+    /// Number of full received-log pages a download may fetch back-to-back before it starts
+    /// pausing between pages. Set to 0 to pace from the first page.
+    pub received_log_pages_before_pacing: usize,
     /// Maximum number of certificate batches downloaded concurrently during chain sync.
     pub max_concurrent_batch_downloads: usize,
     /// Maximum number of tasks that can be joined concurrently using buffer_unordered.
@@ -262,6 +265,7 @@ impl Options {
         use super::{
             DEFAULT_CERTIFICATE_DOWNLOAD_BATCH_SIZE, DEFAULT_CERTIFICATE_UPLOAD_BATCH_SIZE,
             DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS, DEFAULT_MAX_EVENT_STREAM_QUERIES,
+            DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING,
             DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE,
         };
         use crate::DEFAULT_QUORUM_GRACE_PERIOD;
@@ -279,6 +283,7 @@ impl Options {
             certificate_download_batch_size: DEFAULT_CERTIFICATE_DOWNLOAD_BATCH_SIZE,
             certificate_upload_batch_size: DEFAULT_CERTIFICATE_UPLOAD_BATCH_SIZE,
             sender_certificate_download_batch_size: DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE,
+            received_log_pages_before_pacing: DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING,
             max_concurrent_batch_downloads: DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS,
             max_joined_tasks: 100,
             allow_fast_blocks: false,

linera-core/src/client/mod.rs

@@ -177,6 +177,10 @@ pub static DEFAULT_CERTIFICATE_UPLOAD_BATCH_SIZE: usize = 500;
 pub static DEFAULT_SENDER_CERTIFICATE_DOWNLOAD_BATCH_SIZE: usize = 20_000;
 /// Default maximum number of concurrent event stream queries.
 pub static DEFAULT_MAX_EVENT_STREAM_QUERIES: usize = 1000;
+/// Default number of full received-log pages a download may fetch back-to-back before pacing
+/// kicks in. With pages of [`CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES`] entries this lets backlogs
+/// of up to just under 100,000 entries sync at full speed.
+pub static DEFAULT_RECEIVED_LOG_PAGES_BEFORE_PACING: usize = 5;
 /// Default maximum number of certificate batch downloads to run concurrently.
 pub static DEFAULT_MAX_CONCURRENT_BATCH_DOWNLOADS: usize = 1;
 
@@ -1884,10 +1888,13 @@ impl<Env: Environment> Client<Env> {
         // Retrieve the list of newly received certificates from this validator.
         let mut remote_log = Vec::new();
         let mut num_pages = 0usize;
+        let mut num_full_pages = 0usize;
         loop {
             trace!("get_received_log_from_validator: looping");
             let query = ChainInfoQuery::new(chain_id).with_received_log_excluding_first_n(offset);
+            let page_start = linera_base::time::Instant::now();
             let info = remote_node.handle_chain_info_query(query).await?;
+            let page_duration = page_start.elapsed();
             let received_entries = info.requested_received_log.len();
             offset += received_entries as u64;
             remote_log.extend(info.requested_received_log);
@@ -1908,6 +1915,17 @@ impl<Env: Environment> Client<Env> {
             if received_entries < CHAIN_INFO_MAX_RECEIVED_LOG_ENTRIES {
                 break;
             }
+            // Serving a page makes the validator read one storage row per log entry, so
+            // walking a long backlog page after page can monopolize the validator's
+            // storage and degrade its read latency for everyone else. Once the walk is
+            // clearly a backlog walk, pause between pages for as long as the previous
+            // page took: this caps the walk at roughly half of the validator's serving
+            // capacity and automatically backs off further when the validator slows
+            // down. Short syncs never reach this point and are unaffected.
+            num_full_pages += 1;
+            if num_full_pages >= self.options.received_log_pages_before_pacing {
+                linera_base::time::timer::sleep(page_duration).await;
+            }
         }
 
         trace!(