diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed0de65d..194646f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,37 @@ jobs: env: RISC0_DEV_MODE: "1" RUST_LOG: "info" - run: cargo nextest run --workspace --exclude integration_tests --all-features + run: cargo nextest run --workspace --exclude integration_tests --exclude test_fixtures --all-features + + test-fixtures-tests: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.head_ref }} + + - uses: ./.github/actions/install-system-deps + + - uses: ./.github/actions/install-risc0 + + - name: Install active toolchain + run: rustup install + + - name: Restore Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: ci-rust-cache + save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + + - name: Install nextest + run: cargo install --locked cargo-nextest + + - name: Run test_fixtures tests + env: + RISC0_DEV_MODE: "1" + RUST_LOG: "info" + run: cargo nextest run -p test_fixtures integration-tests-prebuild: runs-on: ubuntu-latest @@ -193,7 +223,7 @@ jobs: echo "Discovered integration targets: $targets_json" integration-tests: - needs: integration-tests-prebuild + needs: [test-fixtures-tests, integration-tests-prebuild] runs-on: ubuntu-latest timeout-minutes: 90 strategy: diff --git a/Cargo.lock b/Cargo.lock index 6de9a8da..f4078371 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8896,6 +8896,7 @@ dependencies = [ "futures", "hex", "humantime-serde", + "itertools 0.14.0", "key_protocol", "lee", "lee_core", @@ -9735,6 +9736,7 @@ name = "test_fixtures" version = "0.1.0" dependencies = [ "anyhow", + "bip39", "bytesize", "common", "env_logger", diff --git a/Justfile b/Justfile index afd3d9b7..41566868 100644 --- a/Justfile +++ b/Justfile @@ -35,6 +35,11 @@ test: @echo "๐Ÿงช Running tests" RISC0_DEV_MODE=1 cargo nextest run --no-fail-fast +# Regenerate the prebuilt sequencer db dump for fast TestContext::new() (needs Docker; commit the dump). +regenerate-test-fixture: + @echo "๐Ÿงช Regenerating test fixtures" + RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture + # Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup). bench: @echo "๐Ÿ“Š Running criterion benches" diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index 2931c833..1e753000 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -39,6 +39,7 @@ url.workspace = true num-bigint.workspace = true risc0-zkvm.workspace = true futures.workspace = true +itertools.workspace = true [features] default = [] diff --git a/lez/sequencer/core/src/block_store.rs b/lez/sequencer/core/src/block_store.rs index aec93d42..77b57661 100644 --- a/lez/sequencer/core/src/block_store.rs +++ b/lez/sequencer/core/src/block_store.rs @@ -10,11 +10,11 @@ use lee::V03State; use lee_core::BlockId; use log::info; use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint; -pub use storage::DbResult; use storage::sequencer::{ RocksDBIO, sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey}, }; +pub use storage::{DbResult, sequencer::DbDump}; pub struct SequencerStore { dbio: Arc, @@ -150,6 +150,28 @@ impl SequencerStore { self.dbio.get_lee_state() } + /// Remove the persisted zone-sdk checkpoint so the next startup is treated as a fresh start. + pub fn delete_zone_checkpoint(&self) -> DbResult<()> { + self.dbio.delete_zone_sdk_checkpoint_bytes() + } + + /// Reset every stored block to `Pending` so the next fresh start republishes the whole chain. + pub fn reset_all_blocks_to_pending(&self) -> DbResult<()> { + self.dbio.reset_all_blocks_to_pending() + } + + /// Single-blob [`DbDump`] of the whole store; restore with [`Self::restore_db_from_dump`]. + pub fn dump(&self) -> DbResult { + self.dbio.dump_all() + } + + /// Create a fresh rocksdb at `location` from `dump`, closing it before returning so a sequencer + /// can open it normally afterwards. + pub fn restore_db_from_dump(location: &Path, dump: &DbDump) -> DbResult<()> { + RocksDBIO::restore_from_dump(location, dump)?; + Ok(()) + } + pub fn get_zone_checkpoint(&self) -> Result> { let Some(bytes) = self.dbio.get_zone_sdk_checkpoint_bytes()? else { return Ok(None); diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 75dcc1a9..6c8d5ce2 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -8,6 +8,7 @@ use common::{ transaction::{LeeTransaction, clock_invocation}, }; use config::{GenesisAction, SequencerConfig}; +use itertools::Itertools as _; use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::GENESIS_BLOCK_ID; use log::{error, info, warn}; @@ -124,7 +125,7 @@ impl SequencerCore { load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); - let (store, state, genesis_block) = Self::open_or_create_store(&config); + let (store, state, _genesis_block) = Self::open_or_create_store(&config); let latest_block_meta = store .latest_block_meta() @@ -151,15 +152,33 @@ impl SequencerCore { .await .expect("Failed to initialize Block Publisher"); - // On a truly fresh start (no checkpoint persisted yet), publish the - // genesis block so the indexer can find the channel start. After the - // first publish, zone-sdk's checkpoint persistence covers further - // restarts. + // Fresh start (no checkpoint): republish all pending blocks if is_fresh_start { - block_publisher - .publish_block(&genesis_block, vec![]) - .await - .expect("Failed to publish genesis block"); + let mut pending_blocks = store + .get_all_blocks() + .filter_ok(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) + .collect::, _>>() + .expect("Failed to read blocks from store while republishing on fresh start"); + pending_blocks.sort_unstable_by_key(|block| block.header.block_id); + + assert!( + pending_blocks + .first() + .is_none_or(|block| block.header.block_id == GENESIS_BLOCK_ID), + "First pending block on fresh start should be the genesis block" + ); + + for block in &pending_blocks { + block_publisher + .publish_block(block, vec![]) + .await + .unwrap_or_else(|err| { + panic!( + "Failed to publish block {} on fresh start: {err:#}", + block.header.block_id + ) + }); + } } let sequencer_core = Self { diff --git a/lez/storage/src/lib.rs b/lez/storage/src/lib.rs index 2edb0ee3..8fd485d3 100644 --- a/lez/storage/src/lib.rs +++ b/lez/storage/src/lib.rs @@ -1,7 +1,7 @@ use rocksdb::{DBWithThreadMode, MultiThreaded, WriteBatch}; use crate::{ - cells::{SimpleReadableCell, SimpleWritableCell}, + cells::{SimpleReadableCell, SimpleStorableCell, SimpleWritableCell}, error::DbError, }; @@ -66,4 +66,17 @@ pub trait DBIO { ) -> DbResult<()> { cell.put_batch(self.db(), params, write_batch) } + + /// Delete a cell. Deleting an absent key is a no-op (rocksdb semantics). + fn del(&self, params: T::KeyParams) -> DbResult<()> { + let cf_ref = T::column_ref(self.db()); + self.db() + .delete_cf(&cf_ref, T::key_constructor(params)?) + .map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some(format!("Failed to delete {:?}", T::CELL_NAME)), + ) + }) + } } diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index 44068517..13f0c378 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -1,12 +1,14 @@ use std::{path::Path, sync::Arc}; +use borsh::{BorshDeserialize, BorshSerialize}; use common::{ HashType, block::{BedrockStatus, Block, BlockMeta}, }; use lee::V03State; use rocksdb::{ - BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, WriteBatch, + BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MultiThreaded, + Options, WriteBatch, }; use crate::{ @@ -44,6 +46,37 @@ pub const DB_LEE_STATE_KEY: &str = "lee_state"; /// Name of state column family. pub const CF_LEE_STATE_NAME: &str = "cf_lee_state"; +/// A single key/value entry from a column family, used inside [`DbDump`]. +#[derive(BorshSerialize, BorshDeserialize)] +pub struct DbDumpEntry { + pub cf_name: String, + pub key: Vec, + pub value: Vec, +} + +/// Schema-agnostic single-blob snapshot of a store: every key/value pair across all column +/// families. Lets a prebuilt store ship as one committed file instead of a rocksdb directory. +#[derive(BorshSerialize, BorshDeserialize)] +pub struct DbDump { + pub entries: Vec, +} + +impl DbDump { + /// Serialize the dump to a borsh blob. + pub fn to_bytes(&self) -> DbResult> { + borsh::to_vec(self).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize DbDump".to_owned())) + }) + } + + /// Deserialize a dump produced by [`Self::to_bytes`]. + pub fn from_bytes(bytes: &[u8]) -> DbResult { + borsh::from_slice(bytes).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to deserialize DbDump".to_owned())) + }) + } +} + pub struct RocksDBIO { pub db: DBWithThreadMode, } @@ -84,6 +117,71 @@ impl RocksDBIO { Ok(dbio) } + /// Dump every key/value pair across all column families into a [`DbDump`]. Column families are + /// discovered from disk, so new ones are captured without a hardcoded list. + pub fn dump_all(&self) -> DbResult { + let cf_names = + DBWithThreadMode::::list_cf(&Options::default(), self.db.path()) + .map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some("Failed to list column families for dump".to_owned()), + ) + })?; + + let mut entries = Vec::new(); + for cf_name in cf_names { + let cf = self.db.cf_handle(&cf_name).ok_or_else(|| { + DbError::db_interaction_error(format!( + "Column family {cf_name:?} listed on disk but not opened; add it to `open_inner`" + )) + })?; + for item in self.db.iterator_cf(&cf, IteratorMode::Start) { + let (key, value) = item.map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some(format!( + "Failed to iterate column family {cf_name:?} for dump" + )), + ) + })?; + entries.push(DbDumpEntry { + cf_name: cf_name.clone(), + key: key.into_vec(), + value: value.into_vec(), + }); + } + } + Ok(DbDump { entries }) + } + + /// Create a fresh rocksdb at `path` populated from a [`DbDump`]. + pub fn restore_from_dump(path: &Path, dump: &DbDump) -> DbResult { + let mut db_opts = Options::default(); + db_opts.create_missing_column_families(true); + db_opts.create_if_missing(true); + let dbio = Self::open_inner(path, &db_opts)?; + + let mut batch = WriteBatch::default(); + for entry in &dump.entries { + let cf = dbio.db.cf_handle(&entry.cf_name).ok_or_else(|| { + DbError::db_interaction_error(format!( + "Unknown column family {:?} in dump", + entry.cf_name + )) + })?; + batch.put_cf(&cf, &entry.key, &entry.value); + } + dbio.db.write(batch).map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some("Failed to write dump restore batch".to_owned()), + ) + })?; + + Ok(dbio) + } + fn open_inner(path: &Path, db_opts: &Options) -> DbResult { let mut cf_opts = Options::default(); cf_opts.set_max_write_buffer_number(16); @@ -246,6 +344,11 @@ impl RocksDBIO { self.put(&ZoneSdkCheckpointCellRef(bytes), ()) } + /// Remove the persisted zone-sdk checkpoint so the next startup is treated as a fresh start. + pub fn delete_zone_sdk_checkpoint_bytes(&self) -> DbResult<()> { + self.del::(()) + } + pub fn get_pending_deposit_events(&self) -> DbResult> { Ok(self .get_opt::(())? @@ -448,10 +551,29 @@ impl RocksDBIO { } pub fn mark_block_as_finalized(&self, block_id: u64) -> DbResult<()> { + self.set_block_bedrock_status(block_id, BedrockStatus::Finalized) + } + + /// Reset every stored block to [`BedrockStatus::Pending`], for snapshotting a store to replay + /// against a fresh Bedrock instance that knows none of the blocks yet. + pub fn reset_all_blocks_to_pending(&self) -> DbResult<()> { + let block_ids: Vec = self + .get_all_blocks() + .filter_map(Result::ok) + .filter(|block| !matches!(block.bedrock_status, BedrockStatus::Pending)) + .map(|block| block.header.block_id) + .collect(); + for id in block_ids { + self.set_block_bedrock_status(id, BedrockStatus::Pending)?; + } + Ok(()) + } + + fn set_block_bedrock_status(&self, block_id: u64, status: BedrockStatus) -> DbResult<()> { let mut block = self.get_block(block_id)?.ok_or_else(|| { DbError::db_interaction_error(format!("Block with id {block_id} not found")) })?; - block.bedrock_status = BedrockStatus::Finalized; + block.bedrock_status = status; let cf_block = self.block_column(); self.db @@ -473,7 +595,7 @@ impl RocksDBIO { .map_err(|rerr| { DbError::rocksdb_cast_message( rerr, - Some(format!("Failed to mark block {block_id} as finalized")), + Some(format!("Failed to set block {block_id} bedrock status")), ) })?; diff --git a/test_fixtures/Cargo.toml b/test_fixtures/Cargo.toml index 71f887b6..6e87dcaa 100644 --- a/test_fixtures/Cargo.toml +++ b/test_fixtures/Cargo.toml @@ -21,6 +21,7 @@ wallet.workspace = true programs.workspace = true anyhow.workspace = true +bip39.workspace = true bytesize.workspace = true env_logger.workspace = true futures.workspace = true diff --git a/test_fixtures/README.md b/test_fixtures/README.md new file mode 100644 index 00000000..502ed8b5 --- /dev/null +++ b/test_fixtures/README.md @@ -0,0 +1,31 @@ +# test_fixtures + +Shared test/bench fixtures that spin up bedrock + sequencer + indexer + wallet end-to-end. + +## Library + +`TestContext` drives the full stack for integration tests and benches: + +```rust +let ctx = TestContext::new().await?; // fast: restores the prebuilt db dump +let ctx = TestContext::builder() + .from_scratch() // apply genesis + claim supply live + .with_genesis(actions) // custom genesis (implies from_scratch) + .disable_indexer() + .build().await?; +``` + +`TestContext::new()` restores `fixtures/prebuilt_sequencer_db.dump` (genesis + initial-supply claim +blocks) instead of re-applying genesis and re-proving the private claims, then syncs the wallet. Use +`from_scratch()` to build genesis live. + +## Binary + +`regenerate_test_fixture` regenerates that dump (needs Docker): + +```sh +just regenerate-test-fixture # RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture +``` + +Rerun and commit `fixtures/prebuilt_sequencer_db.dump` after changing genesis, the default accounts, +block format, or initial state. diff --git a/test_fixtures/fixtures/prebuilt_sequencer_db.dump b/test_fixtures/fixtures/prebuilt_sequencer_db.dump new file mode 100644 index 00000000..e07ec265 Binary files /dev/null and b/test_fixtures/fixtures/prebuilt_sequencer_db.dump differ diff --git a/test_fixtures/src/bin/regenerate_test_fixture.rs b/test_fixtures/src/bin/regenerate_test_fixture.rs new file mode 100644 index 00000000..4405277a --- /dev/null +++ b/test_fixtures/src/bin/regenerate_test_fixture.rs @@ -0,0 +1,124 @@ +//! Regenerate the prebuilt sequencer database dump for the fast `TestContext::new()` path. +//! Needs Docker. Run via `just regenerate-test-fixture`, then commit the dump. + +#![expect(clippy::print_stdout, reason = "It's normal in this small cli")] + +use std::{path::Path, time::Duration}; + +use anyhow::{Context as _, Result}; +use lee::PrivateKey; +use sequencer_core::block_store::SequencerStore; +use test_fixtures::{ + config, + setup::{ + prebuilt_sequencer_db_dump_path, setup_bedrock_node, + setup_private_accounts_with_initial_supply, setup_public_accounts_with_initial_supply, + setup_sequencer, setup_wallet, + }, +}; +use wallet::config::WalletConfigOverrides; + +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + + let dest = prebuilt_sequencer_db_dump_path(); + println!( + "๐Ÿ—ƒ๏ธ Regenerating prebuilt sequencer db fixture at {}", + dest.display() + ); + + generate_prebuilt_fixture(&dest) + .await + .context("Failed to regenerate prebuilt sequencer database fixture")?; + + println!("โœ… Wrote fixture dump to {}", dest.display()); + Ok(()) +} + +/// Run a real sequencer with the default accounts, apply genesis + claim the initial supply +/// (genesis block + claim block), then strip the checkpoint and reset blocks to `Pending` so the +/// dump replays cleanly against a fresh Bedrock. Writes the dump to `dest`. +async fn generate_prebuilt_fixture(dest: &Path) -> Result<()> { + let (_bedrock_compose, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to setup Bedrock node")?; + + let initial_public_accounts = config::default_public_accounts_for_wallet(); + let initial_private_accounts = config::default_private_accounts_for_wallet(); + let genesis = + config::genesis_from_accounts(&initial_public_accounts, &initial_private_accounts); + + let (sequencer_handle, temp_sequencer_dir) = setup_sequencer( + config::SequencerPartialConfig::default(), + bedrock_addr, + genesis, + ) + .await + .context("Failed to setup Sequencer for fixture generation")?; + + let (mut wallet, _temp_wallet_dir, _wallet_password) = setup_wallet( + sequencer_handle.addr(), + &initial_public_accounts, + &initial_private_accounts, + WalletConfigOverrides::default(), + ) + .context("Failed to setup wallet for fixture generation")?; + + setup_public_accounts_with_initial_supply(&mut wallet, &initial_public_accounts) + .await + .context("Failed to initialize public accounts for fixture generation")?; + setup_private_accounts_with_initial_supply(&mut wallet, &initial_private_accounts) + .await + .context("Failed to initialize private accounts for fixture generation")?; + + // Shut down gracefully to release the rocksdb lock before reopening the store. + drop(wallet); + drop(sequencer_handle); + + let db_path = temp_sequencer_dir.path().join("rocksdb"); + let store = open_store_with_retry(&db_path) + .await + .context("Failed to reopen sequencer store after shutdown")?; + store + .delete_zone_checkpoint() + .context("Failed to strip zone-sdk checkpoint from fixture database")?; + store + .reset_all_blocks_to_pending() + .context("Failed to reset fixture blocks to pending")?; + let dump = store.dump().context("Failed to dump fixture database")?; + drop(store); + + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create fixture directory {}", parent.display()))?; + } + let bytes = dump + .to_bytes() + .context("Failed to serialize fixture dump")?; + std::fs::write(dest, bytes) + .with_context(|| format!("Failed to write fixture dump to {}", dest.display()))?; + + Ok(()) +} + +/// Reopen the store, retrying while the shut-down sequencer's aborted tasks are still releasing +/// their db handles (the process holds the file lock until then). Must `await` between attempts. +async fn open_store_with_retry(db_path: &Path) -> Result { + let signing_key = PrivateKey::try_new(config::SEQUENCER_SIGNING_KEY) + .expect("Fixed sequencer signing key must be valid"); + + let mut last_err = None; + for _ in 0..100 { + // Let the runtime drop the aborted sequencer tasks before each attempt. + tokio::time::sleep(Duration::from_millis(100)).await; + match SequencerStore::open_db(db_path, signing_key.clone()) { + Ok(store) => return Ok(store), + Err(err) => last_err = Some(err), + } + } + Err(anyhow::anyhow!( + "Failed to open sequencer store at {} after retries: {last_err:?}", + db_path.display() + )) +} diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index 92ed6a18..dca5d445 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -3,7 +3,7 @@ use std::{net::SocketAddr, path::PathBuf, time::Duration}; use anyhow::{Context as _, Result}; use bytesize::ByteSize; use indexer_service::{ChannelId, ClientConfig, IndexerConfig}; -use key_protocol::key_management::KeyChain; +use key_protocol::key_management::{KeyChain, secret_holders::SeedHolder}; use lee::{AccountId, PrivateKey, PublicKey}; use lee_core::Identifier; use sequencer_core::config::{BedrockConfig, GenesisAction, SequencerConfig}; @@ -13,6 +13,15 @@ use wallet::config::WalletConfig; pub const INITIAL_PUBLIC_BALANCES_FOR_WALLET: [u128; 2] = [10_000, 20_000]; pub const INITIAL_PRIVATE_BALANCES_FOR_WALLET: [u128; 2] = [10_000, 20_000]; +/// Fixed sequencer signing key; exposed so the fixture generator can reopen the produced store. +pub const SEQUENCER_SIGNING_KEY: [u8; 32] = [37; 32]; + +// Fixed entropy seeds for the default accounts: deterministic so one prebuilt database is reusable, +// and distinct from the `testnet_initial_state` accounts to avoid depending on / double-funding +// them. +const DEFAULT_PUBLIC_ACCOUNT_SEEDS: [[u8; 32]; 2] = [[0x11; 32], [0x22; 32]]; +const DEFAULT_PRIVATE_ACCOUNT_SEEDS: [[u8; 32]; 2] = [[0x33; 32], [0x44; 32]]; + #[derive(Clone)] pub struct InitialPrivateAccountForWallet { pub key_chain: KeyChain, @@ -87,7 +96,7 @@ pub fn sequencer_config( block_create_timeout, retry_pending_blocks_timeout: Duration::from_secs(5), genesis: genesis_transactions, - signing_key: [37; 32], + signing_key: SEQUENCER_SIGNING_KEY, bedrock_config: BedrockConfig { channel_id: bedrock_channel_id(), node_url: addr_to_url(UrlProtocol::Http, bedrock_addr) @@ -99,7 +108,10 @@ pub fn sequencer_config( #[must_use] pub fn default_public_accounts_for_wallet() -> Vec<(PrivateKey, u128)> { - let mut private_keys = vec![PrivateKey::new_os_random(), PrivateKey::new_os_random()]; + let mut private_keys = DEFAULT_PUBLIC_ACCOUNT_SEEDS + .iter() + .map(|seed| PrivateKey::try_new(*seed).expect("Fixed public account seed must be valid")) + .collect::>(); private_keys.sort_unstable_by_key(|private_key| { AccountId::from(&PublicKey::new_from_private_key(private_key)) }); @@ -112,7 +124,10 @@ pub fn default_public_accounts_for_wallet() -> Vec<(PrivateKey, u128)> { #[must_use] pub fn default_private_accounts_for_wallet() -> Vec { - let mut key_chains = vec![KeyChain::new_os_random(), KeyChain::new_os_random()]; + let mut key_chains = DEFAULT_PRIVATE_ACCOUNT_SEEDS + .iter() + .map(|seed| deterministic_private_key_chain(*seed)) + .collect::>(); key_chains.sort_unstable(); key_chains @@ -126,6 +141,25 @@ pub fn default_private_accounts_for_wallet() -> Vec KeyChain { + let mnemonic = + bip39::Mnemonic::from_entropy(&entropy).expect("32 bytes of entropy is valid for bip39"); + let seed_holder = SeedHolder::from_mnemonic(&mnemonic, ""); + + let secret_spending_key = seed_holder.produce_top_secret_key_holder(); + let private_key_holder = secret_spending_key.produce_private_key_holder(None); + let nullifier_public_key = private_key_holder.generate_nullifier_public_key(); + let viewing_public_key = private_key_holder.generate_viewing_public_key(); + + KeyChain { + secret_spending_key, + private_key_holder, + nullifier_public_key, + viewing_public_key, + } +} + #[must_use] pub fn genesis_from_accounts( public_accounts: &[(PrivateKey, u128)], diff --git a/test_fixtures/src/lib.rs b/test_fixtures/src/lib.rs index e57e2f74..adbb07cb 100644 --- a/test_fixtures/src/lib.rs +++ b/test_fixtures/src/lib.rs @@ -25,7 +25,8 @@ use crate::{ indexer_client::IndexerClient, setup::{ setup_bedrock_node, setup_indexer, setup_private_accounts_with_initial_supply, - setup_public_accounts_with_initial_supply, setup_sequencer, setup_wallet, + setup_public_accounts_with_initial_supply, setup_sequencer, setup_sequencer_from_prebuilt, + setup_wallet, sync_wallet_from_prebuilt, }, }; @@ -254,6 +255,7 @@ pub struct TestContextBuilder { sequencer_partial_config: Option, enable_indexer: bool, wallet_config_overrides: WalletConfigOverrides, + from_scratch: bool, } impl TestContextBuilder { @@ -263,6 +265,7 @@ impl TestContextBuilder { sequencer_partial_config: None, enable_indexer: true, wallet_config_overrides: WalletConfigOverrides::default(), + from_scratch: false, } } @@ -291,6 +294,14 @@ impl TestContextBuilder { self } + /// Build from genesis live instead of loading the prebuilt fixture. Implied by + /// [`Self::with_genesis`]. + #[must_use] + pub const fn from_scratch(mut self) -> Self { + self.from_scratch = true; + self + } + /// Exclude Indexer from test context. /// Indexer is enabled by default. /// @@ -308,6 +319,7 @@ impl TestContextBuilder { sequencer_partial_config, enable_indexer, wallet_config_overrides, + from_scratch, } = self; // Ensure logger is initialized only once @@ -315,6 +327,10 @@ impl TestContextBuilder { debug!("Test context setup"); + // The fixture bakes in the default accounts + genesis, so custom genesis / from_scratch + // must build live. Otherwise load the fixture (fails if it is missing). + let use_prebuilt = !from_scratch && genesis_transactions.is_none(); + let (bedrock_compose, bedrock_addr) = setup_bedrock_node() .await .context("Failed to setup Bedrock node")?; @@ -339,25 +355,30 @@ impl TestContextBuilder { let initial_public_accounts = config::default_public_accounts_for_wallet(); let initial_private_accounts = config::default_private_accounts_for_wallet(); - // Wallet genesis must always be present so that - // setup_public/private_accounts_with_initial_supply can claim from the vault PDAs. - // When a test supplies custom genesis, merge rather than replace. - let wallet_genesis = - config::genesis_from_accounts(&initial_public_accounts, &initial_private_accounts); - let genesis = match genesis_transactions { - Some(mut custom) => { - custom.extend(wallet_genesis); - custom - } - None => wallet_genesis, + + let partial_config = sequencer_partial_config.unwrap_or_default(); + + let (sequencer_handle, temp_sequencer_dir) = if use_prebuilt { + setup_sequencer_from_prebuilt(partial_config, bedrock_addr) + .await + .context("Failed to setup Sequencer from prebuilt database")? + } else { + // Wallet genesis must always be present so that + // setup_public/private_accounts_with_initial_supply can claim from the vault PDAs. + // When a test supplies custom genesis, merge rather than replace. + let wallet_genesis = + config::genesis_from_accounts(&initial_public_accounts, &initial_private_accounts); + let genesis = match genesis_transactions { + Some(mut custom) => { + custom.extend(wallet_genesis); + custom + } + None => wallet_genesis, + }; + setup_sequencer(partial_config, bedrock_addr, genesis) + .await + .context("Failed to setup Sequencer")? }; - let (sequencer_handle, temp_sequencer_dir) = setup_sequencer( - sequencer_partial_config.unwrap_or_default(), - bedrock_addr, - genesis, - ) - .await - .context("Failed to setup Sequencer")?; let (mut wallet, temp_wallet_dir, wallet_password) = setup_wallet( sequencer_handle.addr(), @@ -367,13 +388,20 @@ impl TestContextBuilder { ) .context("Failed to setup wallet")?; - setup_public_accounts_with_initial_supply(&mut wallet, &initial_public_accounts) - .await - .context("Failed to initialize public accounts in wallet")?; + if use_prebuilt { + // Funds already exist on-chain in the prebuilt blocks; sync instead of claiming live. + sync_wallet_from_prebuilt(&mut wallet) + .await + .context("Failed to sync wallet from prebuilt database")?; + } else { + setup_public_accounts_with_initial_supply(&mut wallet, &initial_public_accounts) + .await + .context("Failed to initialize public accounts in wallet")?; - setup_private_accounts_with_initial_supply(&mut wallet, &initial_private_accounts) - .await - .context("Failed to initialize private accounts in wallet")?; + setup_private_accounts_with_initial_supply(&mut wallet, &initial_private_accounts) + .await + .context("Failed to initialize private accounts in wallet")?; + } let sequencer_url = config::addr_to_url(config::UrlProtocol::Http, sequencer_handle.addr()) .context("Failed to convert sequencer addr to URL")?; @@ -405,6 +433,7 @@ impl TestContextBuilder { }) } } + /// A test context to be used in normal #[test] tests. pub struct BlockingTestContext { ctx: Option, diff --git a/test_fixtures/src/setup.rs b/test_fixtures/src/setup.rs index f7d194e3..de5ee58d 100644 --- a/test_fixtures/src/setup.rs +++ b/test_fixtures/src/setup.rs @@ -4,6 +4,7 @@ use anyhow::{Context as _, Result, bail}; use indexer_service::IndexerHandle; use lee::{AccountId, PrivateKey, PublicKey}; use log::{debug, warn}; +use sequencer_core::block_store::{DbDump, SequencerStore}; use sequencer_service::{GenesisAction, SequencerHandle}; use tempfile::TempDir; use testcontainers::compose::DockerCompose; @@ -19,6 +20,28 @@ use crate::{ private_mention, public_mention, }; +/// How to initialize the sequencer's database. +pub enum SequencerInit<'dump> { + /// Apply these genesis actions from scratch. + Genesis(Vec), + /// Restore an existing chain from a prebuilt dump (genesis actions are then irrelevant). + Prebuilt(&'dump DbDump), +} + +/// Committed single-file dump of the prebuilt sequencer database (`just regenerate-test-fixture`). +#[must_use] +pub fn prebuilt_sequencer_db_dump_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/prebuilt_sequencer_db.dump") +} + +/// Load and deserialize the committed prebuilt-database dump. +fn load_prebuilt_dump() -> Result { + let path = prebuilt_sequencer_db_dump_path(); + let bytes = std::fs::read(&path) + .with_context(|| format!("Failed to read prebuilt db dump at {}", path.display()))?; + DbDump::from_bytes(&bytes).context("Failed to deserialize prebuilt db dump") +} + pub async fn setup_bedrock_node() -> Result<(DockerCompose, SocketAddr)> { let manifest_dir = env!("CARGO_MANIFEST_DIR"); let bedrock_compose_path = PathBuf::from(manifest_dir).join("../bedrock/docker-compose.yml"); @@ -111,6 +134,29 @@ pub async fn setup_sequencer( partial: config::SequencerPartialConfig, bedrock_addr: SocketAddr, genesis_transactions: Vec, +) -> Result<(SequencerHandle, TempDir)> { + setup_sequencer_inner( + partial, + bedrock_addr, + SequencerInit::Genesis(genesis_transactions), + ) + .await +} + +/// Like [`setup_sequencer`], but rebuilds the sequencer database from the committed prebuilt dump +/// so it starts from an existing chain instead of applying genesis from scratch. +pub async fn setup_sequencer_from_prebuilt( + partial: config::SequencerPartialConfig, + bedrock_addr: SocketAddr, +) -> Result<(SequencerHandle, TempDir)> { + let dump = load_prebuilt_dump()?; + setup_sequencer_inner(partial, bedrock_addr, SequencerInit::Prebuilt(&dump)).await +} + +async fn setup_sequencer_inner( + partial: config::SequencerPartialConfig, + bedrock_addr: SocketAddr, + init: SequencerInit<'_>, ) -> Result<(SequencerHandle, TempDir)> { let temp_sequencer_dir = tempfile::tempdir().context("Failed to create temp dir for sequencer home")?; @@ -120,6 +166,17 @@ pub async fn setup_sequencer( temp_sequencer_dir.path().display() ); + let genesis_transactions = match init { + SequencerInit::Genesis(genesis) => genesis, + SequencerInit::Prebuilt(dump) => { + // `SequencerCore::open_or_create_store` looks for `/rocksdb`. + let dst = temp_sequencer_dir.path().join("rocksdb"); + SequencerStore::restore_db_from_dump(&dst, dump) + .context("Failed to restore prebuilt sequencer database from dump")?; + Vec::new() + } + }; + let config = config::sequencer_config( partial, temp_sequencer_dir.path().to_owned(), @@ -224,6 +281,15 @@ pub async fn setup_private_accounts_with_initial_supply( Ok(()) } +pub async fn sync_wallet_from_prebuilt(wallet: &mut WalletCore) -> Result<()> { + wallet + .sync_to_latest_block() + .await + .context("Failed to sync wallet from prebuilt chain")?; + + Ok(()) +} + async fn claim_funds_from_vault_to_private( wallet: &mut WalletCore, owner_id: AccountId, diff --git a/test_fixtures/tests/prebuilt_fixture.rs b/test_fixtures/tests/prebuilt_fixture.rs new file mode 100644 index 00000000..4960af81 --- /dev/null +++ b/test_fixtures/tests/prebuilt_fixture.rs @@ -0,0 +1,58 @@ +//! "Genesis follows config" checks for both the from-scratch and prebuilt `TestContext` paths. +//! Need Docker/Bedrock, like the integration tests. +#![expect(clippy::tests_outside_test_module, reason = "It's a tests module")] + +use anyhow::{Context as _, Result}; +use lee::{AccountId, PublicKey}; +use sequencer_service_rpc::RpcClient as _; +use test_fixtures::{ + TestContext, + config::{default_private_accounts_for_wallet, default_public_accounts_for_wallet}, + verify_commitment_is_in_state, +}; + +/// Builds from genesis (no prebuilt database) and checks the on-chain state follows the config. +#[tokio::test] +async fn genesis_from_scratch_follows_config() -> Result<()> { + let ctx = TestContext::builder().from_scratch().build().await?; + assert_context_follows_config(&ctx).await +} + +/// Loads the prebuilt database via [`TestContext::new`] and checks the restored state follows the +/// config โ€” i.e. the dump round-trips and all expected data persists. +#[tokio::test] +async fn prebuilt_context_follows_config() -> Result<()> { + let ctx = TestContext::new().await?; + assert_context_follows_config(&ctx).await +} + +/// Assert every configured default account is funded with its configured balance, and every +/// configured private account's commitment is present in sequencer state. (The wallet also has an +/// unfunded default root account, so we check the configured accounts specifically.) +async fn assert_context_follows_config(ctx: &TestContext) -> Result<()> { + for (private_key, expected_balance) in default_public_accounts_for_wallet() { + let account_id = AccountId::from(&PublicKey::new_from_private_key(&private_key)); + let balance = ctx + .sequencer_client() + .get_account_balance(account_id) + .await?; + assert_eq!( + balance, expected_balance, + "public account {account_id} balance" + ); + } + + for account in default_private_accounts_for_wallet() { + let account_id = account.account_id(); + let commitment = ctx + .wallet() + .get_private_account_commitment(account_id) + .with_context(|| format!("commitment for private account {account_id}"))?; + assert!( + verify_commitment_is_in_state(commitment, ctx.sequencer_client()).await, + "private commitment for {account_id} should be in sequencer state" + ); + } + + Ok(()) +}