fix(sequencer): rework WithdrawalReconciliationKey by using released note id

This commit is contained in:
Daniil Polyakov 2026-07-31 00:43:16 +03:00
parent b9a0ebef6c
commit 7e283f5fed
10 changed files with 179 additions and 135 deletions

1
Cargo.lock generated
View File

@ -4325,7 +4325,6 @@ dependencies = [
"logos-blockchain-http-api-common",
"logos-blockchain-key-management-system-service",
"logos-blockchain-zone-sdk",
"num-bigint 0.4.6",
"ping_core",
"programs",
"reqwest",

View File

@ -51,4 +51,3 @@ tempfile.workspace = true
bytesize.workspace = true
reqwest.workspace = true
borsh.workspace = true
num-bigint.workspace = true

View File

@ -4,7 +4,7 @@
reason = "We don't care about these in tests"
)]
use std::{ops::Deref as _, time::Duration};
use std::{collections::HashSet, ops::Deref as _, time::Duration};
use anyhow::Context as _;
use borsh::BorshSerialize;
@ -31,7 +31,6 @@ use logos_blockchain_http_api_common::bodies::{
use logos_blockchain_zone_sdk::{
CommonHttpClient, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer,
};
use num_bigint::BigUint;
use sequencer_service_rpc::RpcClient as _;
use test_fixtures::public_mention;
use tokio::test;
@ -269,24 +268,7 @@ async fn submit_bedrock_deposit(
let channel_id = integration_tests::config::bedrock_channel_id();
let client = reqwest::Client::new();
let query_balance = || async {
let balance_response = client
.get(format!(
"http://{bedrock_addr}/wallet/{bedrock_account_pk}/balance"
))
.send()
.await
.context("Failed to query Bedrock wallet balance")?;
let balance_response = check_response_success(balance_response).await?;
balance_response
.json::<WalletBalanceResponseBody>()
.await
.context("Failed to decode Bedrock balance response")
};
let mut balance = query_balance().await?;
let mut balance = bedrock_wallet_balance(bedrock_addr, bedrock_account_pk).await?;
info!(
"Queried Bedrock balance for key {bedrock_account_pk}: {:?}",
@ -338,7 +320,7 @@ async fn submit_bedrock_deposit(
let mut found_note = None;
for _ in 0..20 {
tokio::time::sleep(Duration::from_millis(500)).await;
balance = query_balance().await?;
balance = bedrock_wallet_balance(bedrock_addr, bedrock_account_pk).await?;
found_note = balance
.notes
.iter()
@ -389,6 +371,27 @@ async fn submit_bedrock_deposit(
Ok(())
}
/// The Bedrock wallet state of `bedrock_account_pk`: its total balance and the
/// notes it owns, keyed by note id.
async fn bedrock_wallet_balance(
bedrock_addr: std::net::SocketAddr,
bedrock_account_pk: &str,
) -> anyhow::Result<WalletBalanceResponseBody> {
let response = reqwest::Client::new()
.get(format!(
"http://{bedrock_addr}/wallet/{bedrock_account_pk}/balance"
))
.send()
.await
.context("Failed to query Bedrock wallet balance")?;
check_response_success(response)
.await?
.json::<WalletBalanceResponseBody>()
.await
.context("Failed to decode Bedrock balance response")
}
async fn check_response_success(response: reqwest::Response) -> anyhow::Result<reqwest::Response> {
if response.status().is_success() {
Ok(response)
@ -530,7 +533,8 @@ async fn bedrock_deposit_claim_and_withdraw_round_trip_succeeds() -> anyhow::Res
let sender_id = recipient_id;
let observer = create_zone_indexer_observer(ctx.bedrock_addr())?;
let observe_fut = wait_for_finalized_withdraw_op(&observer, amount, bedrock_account_pk);
let observe_fut =
wait_for_finalized_withdraw_op(&observer, ctx.bedrock_addr(), amount, bedrock_account_pk);
let withdraw_fut = execute_subcommand(
ctx.wallet_mut(),
@ -571,22 +575,27 @@ fn create_zone_indexer_observer(
))
}
/// Waits for a finalized withdraw that pays `expected_amount` to `receiver_pk`.
///
/// A withdraw op releases channel-owned notes and carries nothing but their
/// ids — the value and recipient live in the note itself. A released note keeps
/// its id, value and public key, so the pairing is checked on the receiver's
/// Bedrock wallet: one of the released notes must land there with the expected
/// value.
async fn wait_for_finalized_withdraw_op(
observer: &ZoneIndexer<NodeHttpClient>,
bedrock_addr: std::net::SocketAddr,
expected_amount: u64,
receiver_pk: &str,
) -> anyhow::Result<()> {
let timeout = TIME_TO_FINALIZE_DEPOSIT_EVENT_ON_BEDROCK
+ Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS);
let bedrock_account_pk_bytes = hex::decode(receiver_pk)
.context("Failed to decode expected receiver public key from hex")?;
let expected_receiver_pk =
logos_blockchain_key_management_system_service::keys::ZkPublicKey::from(
BigUint::from_bytes_le(&bedrock_account_pk_bytes),
);
tokio::time::timeout(timeout, async {
// The wallet can trail the channel event, so released notes accumulate
// across polls instead of being checked once when first observed.
let mut released_notes = HashSet::new();
loop {
let stream = observer
.follow()
@ -597,20 +606,17 @@ async fn wait_for_finalized_withdraw_op(
while let Some(message) = stream.next().await {
info!("Observed zone message {message:?}");
let ZoneMessage::Withdraw(withdraw) = message else {
continue;
};
let mut iter = withdraw.outputs.iter();
let Some(note) = iter.next() else {
continue;
};
if iter.next().is_some() {
// Withdraw op should only have one output
continue;
if let ZoneMessage::Withdraw(withdraw) = message {
released_notes.extend(withdraw.inputs.iter().copied());
}
}
if note.value == expected_amount && note.pk == expected_receiver_pk {
if !released_notes.is_empty() {
let balance = bedrock_wallet_balance(bedrock_addr, receiver_pk).await?;
if released_notes
.iter()
.any(|note_id| balance.notes.get(note_id) == Some(&expected_amount))
{
return Ok(());
}
}

View File

@ -4,7 +4,10 @@ use anyhow::{Context as _, Result, anyhow, ensure};
use common::block::Block;
use futures::Stream;
use log::{info, warn};
pub use logos_blockchain_core::mantle::ops::channel::{Ed25519PublicKey, MsgId};
pub use logos_blockchain_core::mantle::{
ledger::NoteId,
ops::channel::{Ed25519PublicKey, MsgId},
};
use logos_blockchain_core::{
mantle::{
MantleTx, SignedMantleTx,
@ -30,7 +33,7 @@ use logos_blockchain_zone_sdk::{
adapter::{Node as _, NodeHttpClient},
indexer::ZoneIndexer,
sequencer::{
ChannelUpdateTx, DepositInfo, Event, FinalizedOp, InscriptionInfo,
ChannelUpdateTx, DepositInfo, Event, FinalizedOp, InscriptionInfo, PendingTx,
SequencerConfig as ZoneSdkSequencerConfig, TurnNotification, WithdrawArg, WithdrawInfo,
ZoneSequencer,
},
@ -73,14 +76,29 @@ pub struct FollowUpdate {
/// persist the whole event in one write.
pub type OnFollowSink = Box<dyn Fn(FollowUpdate) + Send + 'static>;
/// What one publish produced.
pub struct PublishOutcome {
/// The `MsgId` zone-sdk assigned the published inscription.
pub this_msg: MsgId,
/// The checkpoint that now holds the inscription as pending.
pub checkpoint: SequencerCheckpoint,
/// Channel notes the bundled withdrawals release, empty for a plain
/// publish.
/// A [`ChannelWithdrawOp`](logos_blockchain_core::mantle::ops::channel::withdraw::ChannelWithdrawOp)
/// carries nothing but the note ids it releases, so these are the only
/// handle the local withdraw intent shares with the Bedrock Withdraw event
/// that later reports it.
pub released_notes: Vec<NoteId>,
}
/// Commands the drive task executes with `&mut sequencer`.
enum Command {
/// Publish an inscription (+ atomic withdrawals); responds with the assigned
/// `MsgId` and the checkpoint that now includes it as pending.
/// Publish an inscription (+ atomic withdrawals); responds with the
/// [`PublishOutcome`].
Publish {
inscription: Inscription,
withdrawals: Vec<WithdrawArg>,
resp: oneshot::Sender<Result<(MsgId, SequencerCheckpoint)>>,
resp: oneshot::Sender<Result<PublishOutcome>>,
},
}
@ -96,9 +114,8 @@ pub trait BlockPublisherTrait: Sized {
on_follow: OnFollowSink,
) -> Result<Self>;
/// Publish a block and return the `MsgId` zone-sdk assigned its inscription
/// together with the checkpoint that now holds it as pending. Zone-sdk
/// drives the actual submission and retries internally.
/// Publish a block and return what zone-sdk made of it. Zone-sdk drives the
/// actual submission and retries internally.
///
/// The checkpoint must be persisted with the block — restoring an older one
/// drops the inscription from the pending set, and it is never resubmitted.
@ -106,7 +123,7 @@ pub trait BlockPublisherTrait: Sized {
&self,
block: &Block,
withdrawals: Vec<WithdrawArg>,
) -> Result<(MsgId, SequencerCheckpoint)>;
) -> Result<PublishOutcome>;
fn channel_id(&self) -> ChannelId;
@ -219,8 +236,11 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
.context("Failed to publish block with withdrawals")
};
let msg_result = published
.map(|(result, checkpoint)| (result.tx.inscription().this_msg, checkpoint));
let msg_result = published.map(|(result, checkpoint)| PublishOutcome {
this_msg: result.tx.inscription().this_msg,
checkpoint,
released_notes: released_notes(&result.tx),
});
match &msg_result {
Ok(_) if withdraw_count == 0 => {
info!("Published block with the size of {data_byte_size} bytes");
@ -326,7 +346,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
&self,
block: &Block,
withdrawals: Vec<WithdrawArg>,
) -> Result<(MsgId, SequencerCheckpoint)> {
) -> Result<PublishOutcome> {
let data = borsh::to_vec(block).context("Failed to serialize block")?;
let data_bounded: Inscription = data
.try_into()
@ -396,6 +416,19 @@ fn block_from_inscription(inscription: &InscriptionInfo) -> Option<(MsgId, Block
.map(|block| (inscription.this_msg, block))
}
/// Channel notes the withdraws bundled with a published tx release; empty for a
/// plain inscription. See [`PublishOutcome::released_notes`].
fn released_notes(tx: &PendingTx) -> Vec<NoteId> {
match tx {
PendingTx::Inscription(_) => Vec::new(),
PendingTx::AtomicWithdraw(bundle) => bundle
.withdraws
.iter()
.flat_map(|withdraw| withdraw.op.inputs.iter().copied())
.collect(),
}
}
/// The inscription carried by an orphaned tx (plain or atomic-withdraw bundle).
const fn channel_update_inscription(orphan: &ChannelUpdateTx) -> Option<&InscriptionInfo> {
match orphan {

View File

@ -41,7 +41,7 @@ use storage::sequencer::{
};
use crate::{
block_publisher::{BlockPublisherTrait, MsgId, ZoneSdkPublisher},
block_publisher::{BlockPublisherTrait, MsgId, NoteId, ZoneSdkPublisher},
block_store::SequencerStore,
task_group::{StoreRelease, TaskGroup},
};
@ -268,7 +268,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let mut last_checkpoint = None;
for block in &pending_blocks {
let (_msg, checkpoint) = block_publisher
let outcome = block_publisher
.publish_block(block, vec![])
.await
.unwrap_or_else(|err| {
@ -277,7 +277,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
block.header.block_id
)
});
last_checkpoint = Some(checkpoint);
last_checkpoint = Some(outcome.checkpoint);
}
// These blocks are already stored, so only the sdk's pending set
@ -542,18 +542,21 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.build_block_from_mempool()
.context("Failed to build block from mempool transactions")?;
let withdrawal_reconciliation_keys: Vec<_> = withdrawals
.iter()
.map(|withdraw| withdraw_event_reconciliation_key(&withdraw.outputs))
.collect::<Result<Vec<_>>>()
.context("Failed to build reconciliation keys for block withdrawals")?;
let (this_msg, checkpoint) = self
let block_publisher::PublishOutcome {
this_msg,
checkpoint,
released_notes,
} = self
.block_publisher
.publish_block(&block, withdrawals)
.await
.context("Failed to publish block to Bedrock")?;
let withdrawal_reconciliation_keys: Vec<_> = released_notes
.iter()
.map(withdrawal_reconciliation_key)
.collect();
self.record_produced_block(
this_msg,
&block,
@ -1146,20 +1149,12 @@ fn apply_follow_update(
let deposit_records: Vec<PendingDepositEventRecord> =
deposits.iter().map(pending_deposit_event_record).collect();
// A withdraw whose outputs we cannot read has no counter to reconcile
// against; log and drop it rather than fail the whole update.
// One reconciliation unit per released note, matching how the intents were
// recorded at publish time.
let consumed_withdrawals: Vec<WithdrawalReconciliationKey> = withdrawals
.iter()
.filter_map(|withdraw| {
withdraw_event_reconciliation_key(&withdraw.op.outputs)
.inspect_err(|err| {
error!(
"Failed to build reconciliation key for Bedrock Withdraw event with tx_hash {}: {err:#}",
hex::encode(withdraw.tx_hash.as_ref())
);
})
.ok()
})
.flat_map(|withdraw| withdraw.op.inputs.iter())
.map(withdrawal_reconciliation_key)
.collect();
// The lock is held across the persist below so disk writes land in apply
@ -1265,9 +1260,8 @@ fn apply_follow_update(
}
for withdrawal in &outcome.unmatched_withdrawals {
warn!(
"Unexpected Bedrock Withdraw event of {} to {}: no matching unseen withdraw found",
withdrawal.amount,
hex::encode(withdrawal.bedrock_account_pk)
"Unexpected Bedrock Withdraw event releasing channel note {}: no matching unseen withdraw found",
hex::encode(withdrawal.released_note_id)
);
}
@ -1643,35 +1637,21 @@ fn extract_bridge_withdraw_data(tx: &LeeTransaction) -> Option<WithdrawArg> {
})
}
fn withdraw_event_reconciliation_key(
outputs: &logos_blockchain_core::mantle::ledger::Outputs,
) -> Result<WithdrawalReconciliationKey> {
let [note] = outputs.as_ref().as_slice() else {
return Err(anyhow!(
"Unsupported withdraw output count for reconciliation: {}",
outputs.len()
));
};
// `extract_bridge_withdraw_data` maps [u8;32] LE -> BigUint -> ZkPublicKey.
// Reconcile by reversing that direction here.
let mut bedrock_account_pk = BigUint::from(note.pk.into_inner()).to_bytes_le();
if bedrock_account_pk.len() > 32 {
return Err(anyhow!(
"Withdraw recipient public key is too large: {} bytes",
bedrock_account_pk.len()
));
}
bedrock_account_pk.resize(32, 0);
let bedrock_account_pk: [u8; 32] = bedrock_account_pk
/// The reconciliation identity of one released channel note.
///
/// A `ChannelWithdrawOp` releases notes the channel already owns and carries
/// only their ids — the recipient key and value live in the note itself, which
/// neither the op nor the Bedrock Withdraw event reports. The note id is
/// therefore the one handle both sides share, and it is unique: a note is spent
/// once.
fn withdrawal_reconciliation_key(note_id: &NoteId) -> WithdrawalReconciliationKey {
let released_note_id: [u8; 32] = note_id
.as_bytes()
.as_ref()
.try_into()
.expect("Public key bytes were padded/truncated to 32 bytes");
.expect("`NoteId` is a 32-byte field element");
Ok(WithdrawalReconciliationKey {
amount: note.value,
bedrock_account_pk,
})
WithdrawalReconciliationKey { released_note_id }
}
/// Load signing key from file or generate a new one if it doesn't exist.

View File

@ -5,14 +5,17 @@ use common::block::Block;
use futures::Stream;
use logos_blockchain_core::{
header::HeaderId,
mantle::ops::channel::{ChannelId, MsgId},
mantle::{
ledger::{NoteId, Utxo},
ops::channel::{ChannelId, MsgId},
},
};
use logos_blockchain_key_management_system_service::keys::Ed25519Key;
use logos_blockchain_zone_sdk::{Slot, ZoneMessage, sequencer::WithdrawArg};
use tokio_util::sync::CancellationToken;
use crate::{
block_publisher::{BlockPublisherTrait, OnFollowSink, SequencerCheckpoint},
block_publisher::{BlockPublisherTrait, OnFollowSink, PublishOutcome, SequencerCheckpoint},
config::BedrockConfig,
};
@ -70,12 +73,16 @@ impl BlockPublisherTrait for MockBlockPublisher {
async fn publish_block(
&self,
block: &Block,
_bridge_withdrawals: Vec<WithdrawArg>,
) -> Result<(MsgId, SequencerCheckpoint)> {
withdrawals: Vec<WithdrawArg>,
) -> Result<PublishOutcome> {
// Deterministic per-block id so head dedup behaves in tests.
//
// TODO: should we allow more "mockability" here?
Ok((MsgId::from(block.header.hash.0), mock_checkpoint()))
Ok(PublishOutcome {
this_msg: MsgId::from(block.header.hash.0),
checkpoint: mock_checkpoint(),
released_notes: mock_released_notes(&withdrawals),
})
}
fn channel_id(&self) -> ChannelId {
@ -108,6 +115,20 @@ impl BlockPublisherTrait for MockBlockPublisher {
}
}
/// The notes the mock reports as released by `withdrawals`.
///
/// Zone-sdk picks the actual channel notes to release, so a mock has to invent
/// them: one note id per requested output, derived from the output itself so
/// tests can recompute the reconciliation keys of a block they produced.
#[must_use]
pub(crate) fn mock_released_notes(withdrawals: &[WithdrawArg]) -> Vec<NoteId> {
withdrawals
.iter()
.flat_map(|withdraw| withdraw.outputs.into_iter().enumerate())
.map(|(output_index, note)| Utxo::new([0; 32], output_index, *note).id())
.collect()
}
/// A zeroed checkpoint, for [`MockBlockPublisher::publish_block`] and for tests
/// building a [`crate::block_publisher::FollowUpdate`]. Tests only assert *that*
/// a checkpoint was persisted alongside its effects, never what is in it.

View File

@ -10,7 +10,7 @@ use chain_state::ChainState;
use common::block::Block;
use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription};
use logos_blockchain_zone_sdk::{Slot, ZoneBlock, ZoneMessage};
use storage::sequencer::sequencer_cells::ZoneAnchorRecord;
use storage::sequencer::sequencer_cells::{WithdrawalReconciliationKey, ZoneAnchorRecord};
use super::*;
use crate::{
@ -320,6 +320,18 @@ fn build_public_withdraw_tx(
LeeTransaction::Public(lee::PublicTransaction::new(message, witness_set))
}
/// The reconciliation key a produced block carries for `withdraw_tx`, keyed on
/// the note [`MockBlockPublisher`] reports as released for it.
fn produced_withdraw_key(withdraw_tx: &LeeTransaction) -> WithdrawalReconciliationKey {
let withdraw_arg = crate::extract_bridge_withdraw_data(withdraw_tx).expect("withdraw data");
let [note_id] = crate::mock::mock_released_notes(std::slice::from_ref(&withdraw_arg))[..]
else {
panic!("A bridge withdraw releases exactly one note");
};
crate::withdrawal_reconciliation_key(&note_id)
}
/// Cold-start backfill re-records an already-finalized deposit event as a
/// pending record before reconstruction replays the same deposit block.
/// Reconstruction must drop that record — its mint is permanently reflected in
@ -443,8 +455,7 @@ async fn reconstructed_deposit_is_not_reminted_after_backfill_redelivery() {
// A reconstructed withdraw's finalized L1 event was already re-delivered (and
// dropped) by cold-start backfill, so it will never be consumed again.
// Reconstruction must not count it, or the count stays phantom-inflated forever.
let withdraw_arg = crate::extract_bridge_withdraw_data(&withdraw_tx).expect("withdraw data");
let key = crate::withdraw_event_reconciliation_key(&withdraw_arg.outputs).expect("recon key");
let key = produced_withdraw_key(&withdraw_tx);
assert!(
!seq_b
.block_store()
@ -482,8 +493,7 @@ async fn reconstructed_withdraw_leaves_no_phantom_unseen_count() {
.unwrap();
seq_a.produce_new_block().await.unwrap();
let withdraw_arg = crate::extract_bridge_withdraw_data(&withdraw_tx).expect("withdraw data");
let key = crate::withdraw_event_reconciliation_key(&withdraw_arg.outputs).expect("recon key");
let key = produced_withdraw_key(&withdraw_tx);
// Producing the withdraw counts it as unseen, awaiting its L1 event.
assert!(
seq_a

View File

@ -775,8 +775,8 @@ impl RocksDBIO {
/// returning one entry per occurrence that matched no local counter.
///
/// Occurrences are folded per key for the same reason as the deposit
/// records: two withdrawals in one update can share a reconciliation key,
/// and a per-occurrence disk read would miss the staged decrement.
/// records: should two withdrawals in one update share a reconciliation
/// key, a per-occurrence disk read would miss the staged decrement.
fn stage_consumed_withdrawals(
&self,
withdrawals: &[WithdrawalReconciliationKey],
@ -853,8 +853,8 @@ impl RocksDBIO {
/// Stages the unseen-withdraw increments for one update into `batch`.
///
/// Occurrences are folded per key for the same reason as
/// [`Self::stage_consumed_withdrawals`]: two intents in one update can share
/// a reconciliation key, and a per-occurrence disk read would miss the
/// [`Self::stage_consumed_withdrawals`]: should two intents in one update
/// share a reconciliation key, a per-occurrence disk read would miss the
/// staged increment and count the pair once.
fn stage_new_withdraw_intents(
&self,

View File

@ -421,10 +421,12 @@ impl SimpleWritableCell for PeerFloorCellRef<'_> {
}
}
/// Identity of one withdrawal, shared by the intent recorded when the
/// sequencer publishes it and the Bedrock Withdraw event that later reports
/// it: the id of the channel note the withdrawal releases.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct WithdrawalReconciliationKey {
pub amount: u64,
pub bedrock_account_pk: [u8; 32],
pub released_note_id: [u8; 32],
}
#[derive(Debug, BorshSerialize, BorshDeserialize)]
@ -437,12 +439,9 @@ impl SimpleStorableCell for UnseenWithdrawCountCell {
const CF_NAME: &'static str = CF_META_NAME;
fn key_constructor(key_params: Self::KeyParams) -> DbResult<Vec<u8>> {
let WithdrawalReconciliationKey {
amount,
bedrock_account_pk,
} = key_params;
let WithdrawalReconciliationKey { released_note_id } = key_params;
borsh::to_vec(&(Self::CELL_NAME, amount, bedrock_account_pk)).map_err(|err| {
borsh::to_vec(&(Self::CELL_NAME, released_note_id)).map_err(|err| {
DbError::borsh_cast_message(
err,
Some(format!(

View File

@ -581,13 +581,11 @@ fn repeated_withdrawal_key_in_one_update_folds_once_per_occurrence() {
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
let key = WithdrawalReconciliationKey {
amount: 7,
bedrock_account_pk: [3; 32],
released_note_id: [3; 32],
};
// Two local intents for the same key in one update — two withdrawals of the
// same amount to the same L1 key. A per-occurrence disk read would miss the
// staged increment and record the pair as one.
// Two local intents for the same key in one update. A per-occurrence disk
// read would miss the staged increment and record the pair as one.
dbio.store_update(&StoreUpdate {
new_withdraw_intents: &[key, key],
..StoreUpdate::new(&state_with_balance(100))
@ -626,8 +624,7 @@ fn unmatched_withdrawal_is_reported_and_writes_nothing() {
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
let key = WithdrawalReconciliationKey {
amount: 5,
bedrock_account_pk: [4; 32],
released_note_id: [4; 32],
};
let outcome = dbio
.store_update(&StoreUpdate {