Skip to content

Steward Writer Service: Larger transaction sizes #69

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aoikurokawa wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Steward Writer Service: Larger transaction sizes #69

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2857058
fix: larger transaction sizes
aoikurokawa Aug 28, 2026
09d4ecc
fix: write example code
aoikurokawa Aug 28, 2026
a9a5eb0
fix: format
aoikurokawa Aug 28, 2026
0ac05ca
fix: example
aoikurokawa Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions core/src/rpc_utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,14 @@ use tokio_retry::strategy::{jitter, FibonacciBackoff};
use tokio_retry::Retry;

pub const MAX_RPC_RETRIES: usize = 10;

/// Highest transaction version we ask the RPC to return.
///
/// `getTransaction` fails outright with `UnsupportedTransactionVersion` when it
/// is asked for a transaction newer than this, so it has to keep up with what
/// the cluster can produce. v1 (SIMD-0296 / SIMD-0385) ships in Agave v4.2.
pub const MAX_SUPPORTED_TRANSACTION_VERSION: u8 = 1;

type RetryStrategy = Take<Map<FibonacciBackoff, fn(Duration) -> Duration>>;

pub fn retry() -> RetryStrategy {
Expand All @@ -21,10 +29,16 @@ pub fn retry() -> RetryStrategy {
.take(MAX_RPC_RETRIES)
}

/// Fetches each signature's transaction, paired with the signature it was
/// requested for.
///
/// Returning the signature lets callers line results up with their own
/// bookkeeping without decoding the transaction, which fails for any version
/// the pinned SDK predates.
pub async fn retry_get_transactions(
rpc_client: &RpcClient,
transaction_signatures: &[Signature],
) -> Result<Vec<EncodedConfirmedTransactionWithStatusMeta>, Box<RpcError>> {
) -> Result<Vec<(Signature, EncodedConfirmedTransactionWithStatusMeta)>, Box<RpcError>> {
let txes = Retry::spawn(retry(), || {
get_signatures_internal(rpc_client, transaction_signatures)
})
Expand All @@ -36,19 +50,19 @@ pub async fn retry_get_transactions(
async fn get_signatures_internal(
rpc_client: &RpcClient,
transaction_signatures: &[Signature],
) -> Result<Vec<EncodedConfirmedTransactionWithStatusMeta>, Box<RpcError>> {
) -> Result<Vec<(Signature, EncodedConfirmedTransactionWithStatusMeta)>, Box<RpcError>> {
let config = RpcTransactionConfig {
commitment: CommitmentConfig::finalized().into(),
encoding: UiTransactionEncoding::Base64.into(),
max_supported_transaction_version: Some(0),
max_supported_transaction_version: Some(MAX_SUPPORTED_TRANSACTION_VERSION),
Comment thread
aoikurokawa marked this conversation as resolved.
};

let mut temp_txs = vec![];
for signature in transaction_signatures.iter() {
let tx = rpc_client
.get_transaction_with_config(signature, config)
.await?;
temp_txs.push(tx);
temp_txs.push((*signature, tx));
}
Ok(temp_txs)
}
Expand Down
117 changes: 72 additions & 45 deletions steward-writer-service/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@ use solana_client::{
nonblocking::rpc_client::RpcClient, rpc_client::GetConfirmedSignaturesForAddress2Config,
rpc_response::RpcConfirmedTransactionStatusWithSignature,
};
use solana_metrics::datapoint_info;
use solana_metrics::{datapoint_error, datapoint_info};
use solana_sdk::{
commitment_config::CommitmentConfig, pubkey::Pubkey, signature::Signature,
transaction::TransactionError,
commitment_config::CommitmentConfig,
pubkey::Pubkey,
signature::Signature,
transaction::{TransactionError, TransactionVersion},
};
use solana_transaction_status::{
option_serializer::OptionSerializer, EncodedConfirmedTransactionWithStatusMeta,
Expand Down Expand Up @@ -143,6 +145,7 @@ async fn main() {
start_slot,
end_slot,
args.dry_run,
&args.cluster_name,
)
.await
{
Expand Down Expand Up @@ -218,8 +221,15 @@ async fn listen(
.first()
.map(|status| Signature::from_str(&status.signature).unwrap());

fetch_and_process_transactions(rpc_client, &rpc_signatures, stake_pool, store, dry_run)
.await?;
fetch_and_process_transactions(
rpc_client,
&rpc_signatures,
stake_pool,
store,
dry_run,
cluster_name,
)
.await?;
}
}
}
Expand All @@ -230,13 +240,8 @@ async fn fetch_and_process_transactions(
stake_pool: &Pubkey,
store: &StewardEventsStore,
dry_run: bool,
) -> Result<
Vec<(
RpcConfirmedTransactionStatusWithSignature,
EncodedConfirmedTransactionWithStatusMeta,
)>,
Box<dyn std::error::Error>,
> {
cluster_name: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let transaction_signatures: Vec<Signature> = signatures
.iter()
.map(|status| Signature::from_str(&status.signature).unwrap())
Expand All @@ -246,23 +251,48 @@ async fn fetch_and_process_transactions(

info!("Fetched {} transactions from rpc", transactions.len());

let mut transaction_data = vec![];
for tx in transactions.into_iter() {
let target_signature = if let Some(tx) = tx.transaction.transaction.decode() {
*tx.signatures.first().unwrap()
} else {
continue;
};
let transaction_status = signatures
.iter()
.find(|status| Signature::from_str(&status.signature).unwrap() == target_signature);
if let Some(status) = transaction_status {
transaction_data.push((status.clone(), tx));
}
}
process_transactions(&transaction_data, stake_pool, store, dry_run).await?;
let transaction_data = pair_with_statuses(signatures, transactions);

process_transactions(&transaction_data, stake_pool, store, dry_run, cluster_name).await
}

/// Pairs fetched transactions back up with the signature statuses they came from.
///
/// `retry_get_transactions` returns one entry per requested signature in request
/// order, so this is a positional zip. The equality check guards against
/// attributing a transaction's events to the wrong signature if that ever stops
/// holding.
fn pair_with_statuses(
signatures: &[RpcConfirmedTransactionStatusWithSignature],
transactions: Vec<(Signature, EncodedConfirmedTransactionWithStatusMeta)>,
) -> Vec<(
RpcConfirmedTransactionStatusWithSignature,
EncodedConfirmedTransactionWithStatusMeta,
)> {
signatures
.iter()
.zip(transactions)
.filter_map(|(status, (signature, tx))| {
if status.signature != signature.to_string() {
error!(
"Signature mismatch: requested {}, RPC returned {signature}",
status.signature
);
return None;
}
Some((status.clone(), tx))
})
.collect()
}

Ok(transaction_data)
/// Human-readable transaction version for diagnostics, including versions this
/// SDK is too old to decode.
fn describe_version(version: Option<&TransactionVersion>) -> String {
match version {
Some(TransactionVersion::Number(n)) => format!("v{n}"),
Some(TransactionVersion::Legacy(_)) => "legacy".to_string(),
None => "unknown-version".to_string(),
}
}

async fn process_transactions(
Expand All @@ -273,6 +303,7 @@ async fn process_transactions(
stake_pool: &Pubkey,
store: &StewardEventsStore,
dry_run: bool,
cluster_name: &str,
) -> Result<(), Box<dyn std::error::Error>> {
// If the slot from `signatures` doesn't match the slot in `transactions`, print it out
for (status, tx) in transactions.iter() {
Expand Down Expand Up @@ -308,7 +339,15 @@ async fn process_transactions(
}
},
None => {
error!("No transaction found in encoded transaction {signature}");
let version = describe_version(transaction.version.as_ref());
error!("Could not decode {version} transaction {signature}, skipping its events");
datapoint_error!(
"steward_writer_service-undecodable_transaction",
("signature", signature.to_string(), String),
("version", version, String),
("slot", *slot as i64, i64),
"cluster" => cluster_name,
);
continue;
}
};
Expand Down Expand Up @@ -362,6 +401,7 @@ async fn process_transactions(

const NUM_TRANSACTIONS: usize = 1000;

#[allow(clippy::too_many_arguments)]
async fn fetch_historical_program_transactions(
program_id: &Pubkey,
rpc_client: &RpcClient,
Expand All @@ -370,6 +410,7 @@ async fn fetch_historical_program_transactions(
start_slot: u64,
end_slot: u64,
dry_run: bool,
cluster_name: &str,
) -> Result<(), Box<dyn std::error::Error>> {
info!("Backfilling transactions between slots {start_slot} and {end_slot}");
let mut before = None;
Expand Down Expand Up @@ -436,23 +477,9 @@ async fn fetch_historical_program_transactions(

let transactions = retry_get_transactions(rpc_client, &transaction_signatures).await?;

// Align transactions with signatures
let mut transaction_data = vec![];
for tx in transactions.into_iter() {
let target_signature = if let Some(tx) = tx.transaction.transaction.decode() {
*tx.signatures.first().unwrap()
} else {
continue;
};
let transaction_status = valid_signatures
.iter()
.find(|status| Signature::from_str(&status.signature).unwrap() == target_signature);
if let Some(status) = transaction_status {
transaction_data.push((status.clone(), tx));
}
}
let transaction_data = pair_with_statuses(&valid_signatures, transactions);

process_transactions(&transaction_data, stake_pool, store, dry_run).await?;
process_transactions(&transaction_data, stake_pool, store, dry_run, cluster_name).await?;

if should_break {
break;
Expand Down
Loading