diff --git a/.deny.toml b/.deny.toml index 4d69e70b..b02f2046 100644 --- a/.deny.toml +++ b/.deny.toml @@ -60,6 +60,7 @@ allow-git = [ "https://github.com/logos-blockchain/logos-blockchain.git", "https://github.com/logos-blockchain/logos-blockchain-circuits.git", "https://github.com/logos-blockchain/logos-blockchain-rust-rapidsnark.git", + "https://github.com/logos-blockchain/sponges", "https://github.com/arkworks-rs/spongefish.git", ] unknown-git = "deny" diff --git a/.github/workflows/bench-regression.yml b/.github/workflows/bench-regression.yml index d5d9568a..d82c0f11 100644 --- a/.github/workflows/bench-regression.yml +++ b/.github/workflows/bench-regression.yml @@ -28,10 +28,6 @@ jobs: - uses: ./.github/actions/install-risc0 - - uses: ./.github/actions/install-logos-blockchain-circuits - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install active toolchain run: rustup install diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed0de65d..194646f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,37 @@ jobs: env: RISC0_DEV_MODE: "1" RUST_LOG: "info" - run: cargo nextest run --workspace --exclude integration_tests --all-features + run: cargo nextest run --workspace --exclude integration_tests --exclude test_fixtures --all-features + + test-fixtures-tests: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.head_ref }} + + - uses: ./.github/actions/install-system-deps + + - uses: ./.github/actions/install-risc0 + + - name: Install active toolchain + run: rustup install + + - name: Restore Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: ci-rust-cache + save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} + + - name: Install nextest + run: cargo install --locked cargo-nextest + + - name: Run test_fixtures tests + env: + RISC0_DEV_MODE: "1" + RUST_LOG: "info" + run: cargo nextest run -p test_fixtures integration-tests-prebuild: runs-on: ubuntu-latest @@ -193,7 +223,7 @@ jobs: echo "Discovered integration targets: $targets_json" integration-tests: - needs: integration-tests-prebuild + needs: [test-fixtures-tests, integration-tests-prebuild] runs-on: ubuntu-latest timeout-minutes: 90 strategy: diff --git a/Cargo.lock b/Cargo.lock index b9d9e453..d6afde36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1943,9 +1943,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -2003,7 +2003,6 @@ dependencies = [ "criterion", "key_protocol", "lee_core", - "rand 0.8.6", ] [[package]] @@ -2166,7 +2165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -3939,9 +3938,7 @@ dependencies = [ "anyhow", "arc-swap", "async-stream", - "authenticated_transfer_core", "borsh", - "bridge_lock_core", "common", "cross_zone", "cross_zone_inbox_core", @@ -3961,6 +3958,7 @@ dependencies = [ "storage", "tempfile", "testnet_initial_state", + "thiserror 2.0.18", "tokio", "url", ] @@ -4008,6 +4006,7 @@ dependencies = [ "base64", "common", "hex", + "indexer_core", "lee", "lee_core", "schemars 1.2.1", @@ -4543,11 +4542,11 @@ dependencies = [ [[package]] name = "keccak" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +source = "git+https://github.com/logos-blockchain/sponges?rev=3a56e99771beedf04946eab21a4a62adc2951377#3a56e99771beedf04946eab21a4a62adc2951377" dependencies = [ "cfg-if", "cpufeatures 0.3.0", + "risc0-zkvm", ] [[package]] @@ -4655,7 +4654,6 @@ dependencies = [ "anyhow", "borsh", "build_utils", - "env_logger", "hex", "hex-literal 1.1.0", "k256", @@ -4670,7 +4668,6 @@ dependencies = [ "test-case", "test_methods", "thiserror 2.0.18", - "token_core", ] [[package]] @@ -9034,6 +9031,7 @@ dependencies = [ "futures", "hex", "humantime-serde", + "itertools 0.14.0", "key_protocol", "lee", "lee_core", @@ -9595,11 +9593,13 @@ dependencies = [ "borsh", "common", "lee", + "log", "programs", "rocksdb", "system_accounts", "tempfile", "thiserror 2.0.18", + "zstd", ] [[package]] @@ -9873,6 +9873,7 @@ name = "test_fixtures" version = "0.1.0" dependencies = [ "anyhow", + "bip39", "bytesize", "common", "env_logger", @@ -9892,6 +9893,7 @@ dependencies = [ "tempfile", "testcontainers", "tokio", + "tokio-util", "url", "wallet", ] @@ -11025,7 +11027,6 @@ dependencies = [ "risc0-zkvm", "sequencer_service_rpc", "serde_json", - "tempfile", "tokio", "vault_core", "wallet", @@ -11954,3 +11955,31 @@ dependencies = [ "log", "simd-adler32", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 99ad6879..164b64cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,6 +161,7 @@ bip39 = "2.2.0" hmac-sha512 = "1.1.7" chrono = "0.4.41" borsh = "1.5.7" +zstd = "0.13" base58 = "0.2.0" itertools = "0.14.0" num-bigint = "0.4.6" @@ -209,6 +210,10 @@ opt-level = 'z' lto = true codegen-units = 1 +# Keccak speedup for in-guest ML KEM +[patch.crates-io] +keccak = { git = "https://github.com/logos-blockchain/sponges", rev = "3a56e99771beedf04946eab21a4a62adc2951377" } + # Keep backtraces but drop full DWARF type info to avoid LLD OOM/SIGBUS when # linking large integration-test binaries on resource-constrained CI runners. [profile.dev] diff --git a/Justfile b/Justfile index 8b5501aa..cf7c833e 100644 --- a/Justfile +++ b/Justfile @@ -40,6 +40,11 @@ test: @echo "๐Ÿงช Running tests" RISC0_DEV_MODE=1 cargo nextest run --no-fail-fast +# Regenerate the prebuilt sequencer db dump for fast TestContext::new() (needs Docker; commit the dump). +regenerate-test-fixture: + @echo "๐Ÿงช Regenerating test fixtures" + RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture + # Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup). bench: @echo "๐Ÿ“Š Running criterion benches" diff --git a/artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin b/artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin index 2b6bee3f..bacba91e 100644 Binary files a/artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin and b/artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin differ diff --git a/artifacts/lez/programs/amm.bin b/artifacts/lez/programs/amm.bin index 333496a2..d86dae93 100644 Binary files a/artifacts/lez/programs/amm.bin and b/artifacts/lez/programs/amm.bin differ diff --git a/artifacts/lez/programs/associated_token_account.bin b/artifacts/lez/programs/associated_token_account.bin index 693f7bef..03e5b252 100644 Binary files a/artifacts/lez/programs/associated_token_account.bin and b/artifacts/lez/programs/associated_token_account.bin differ diff --git a/artifacts/lez/programs/authenticated_transfer.bin b/artifacts/lez/programs/authenticated_transfer.bin index f4566fe1..9a23257d 100644 Binary files a/artifacts/lez/programs/authenticated_transfer.bin and b/artifacts/lez/programs/authenticated_transfer.bin differ diff --git a/artifacts/lez/programs/bridge.bin b/artifacts/lez/programs/bridge.bin index fe4ab835..d9ad7282 100644 Binary files a/artifacts/lez/programs/bridge.bin and b/artifacts/lez/programs/bridge.bin differ diff --git a/artifacts/lez/programs/bridge_lock.bin b/artifacts/lez/programs/bridge_lock.bin index 77d68b6c..224d182e 100644 Binary files a/artifacts/lez/programs/bridge_lock.bin and b/artifacts/lez/programs/bridge_lock.bin differ diff --git a/artifacts/lez/programs/clock.bin b/artifacts/lez/programs/clock.bin index 15745397..0fa4fbae 100644 Binary files a/artifacts/lez/programs/clock.bin and b/artifacts/lez/programs/clock.bin differ diff --git a/artifacts/lez/programs/cross_zone_inbox.bin b/artifacts/lez/programs/cross_zone_inbox.bin index 4b0d9a70..376f55db 100644 Binary files a/artifacts/lez/programs/cross_zone_inbox.bin and b/artifacts/lez/programs/cross_zone_inbox.bin differ diff --git a/artifacts/lez/programs/cross_zone_outbox.bin b/artifacts/lez/programs/cross_zone_outbox.bin index 6727034a..c7814a7e 100644 Binary files a/artifacts/lez/programs/cross_zone_outbox.bin and b/artifacts/lez/programs/cross_zone_outbox.bin differ diff --git a/artifacts/lez/programs/faucet.bin b/artifacts/lez/programs/faucet.bin index ffb432af..a14dbbf2 100644 Binary files a/artifacts/lez/programs/faucet.bin and b/artifacts/lez/programs/faucet.bin differ diff --git a/artifacts/lez/programs/pinata.bin b/artifacts/lez/programs/pinata.bin index 7588e93b..8dd35ccd 100644 Binary files a/artifacts/lez/programs/pinata.bin and b/artifacts/lez/programs/pinata.bin differ diff --git a/artifacts/lez/programs/pinata_token.bin b/artifacts/lez/programs/pinata_token.bin index fd47cfe6..bfec3f77 100644 Binary files a/artifacts/lez/programs/pinata_token.bin and b/artifacts/lez/programs/pinata_token.bin differ diff --git a/artifacts/lez/programs/ping_receiver.bin b/artifacts/lez/programs/ping_receiver.bin index 83154cf2..7b6390b0 100644 Binary files a/artifacts/lez/programs/ping_receiver.bin and b/artifacts/lez/programs/ping_receiver.bin differ diff --git a/artifacts/lez/programs/ping_sender.bin b/artifacts/lez/programs/ping_sender.bin index 9ba41f86..bd70e8a6 100644 Binary files a/artifacts/lez/programs/ping_sender.bin and b/artifacts/lez/programs/ping_sender.bin differ diff --git a/artifacts/lez/programs/token.bin b/artifacts/lez/programs/token.bin index ae0c7bc5..2a515a03 100644 Binary files a/artifacts/lez/programs/token.bin and b/artifacts/lez/programs/token.bin differ diff --git a/artifacts/lez/programs/vault.bin b/artifacts/lez/programs/vault.bin index fd9d0878..87829416 100644 Binary files a/artifacts/lez/programs/vault.bin and b/artifacts/lez/programs/vault.bin differ diff --git a/artifacts/lez/programs/wrapped_token.bin b/artifacts/lez/programs/wrapped_token.bin index 472040d4..c775919c 100644 Binary files a/artifacts/lez/programs/wrapped_token.bin and b/artifacts/lez/programs/wrapped_token.bin differ diff --git a/integration_tests/tests/account.rs b/integration_tests/tests/account.rs index dd6ba4ed..2b69f7e0 100644 --- a/integration_tests/tests/account.rs +++ b/integration_tests/tests/account.rs @@ -141,7 +141,11 @@ async fn import_private_account() -> Result<()> { let mut ctx = TestContext::new().await?; let key_chain = KeyChain::new_os_random(); - let account_id = lee::AccountId::from((&key_chain.nullifier_public_key, 0)); + let account_id = lee::AccountId::from(( + &key_chain.nullifier_public_key, + &key_chain.viewing_public_key, + 0, + )); let account = lee::Account { program_owner: programs::authenticated_transfer().id(), balance: 777, @@ -198,7 +202,11 @@ async fn import_private_account_second_time_overrides_account_data() -> Result<( let mut ctx = TestContext::new().await?; let key_chain = KeyChain::new_os_random(); - let account_id = lee::AccountId::from((&key_chain.nullifier_public_key, 0)); + let account_id = lee::AccountId::from(( + &key_chain.nullifier_public_key, + &key_chain.viewing_public_key, + 0, + )); let key_chain_json = serde_json::to_string(&key_chain).context("Failed to serialize key chain")?; diff --git a/integration_tests/tests/auth_transfer/private.rs b/integration_tests/tests/auth_transfer/private.rs index b543a2f6..af016dc6 100644 --- a/integration_tests/tests/auth_transfer/private.rs +++ b/integration_tests/tests/auth_transfer/private.rs @@ -8,15 +8,14 @@ use integration_tests::{ private_mention, public_mention, send, sync_private, verify_commitment_is_in_state, }; use lee::{ - AccountId, SharedSecretKey, execute_and_prove, - privacy_preserving_transaction::circuit::ProgramWithDependencies, program::Program, + AccountId, execute_and_prove, privacy_preserving_transaction::circuit::ProgramWithDependencies, + program::Program, }; use lee_core::{ - DUMMY_COMMITMENT, DUMMY_COMMITMENT_HASH, EncryptedAccountData, InputAccountIdentity, - ML_KEM_768_CIPHERTEXT_LEN, Nullifier, NullifierPublicKey, + DUMMY_COMMITMENT, DUMMY_COMMITMENT_HASH, InputAccountIdentity, Nullifier, NullifierPublicKey, account::{Account, AccountWithMetadata}, compute_digest_for_path, - encryption::{EphemeralPublicKey, ViewingPublicKey}, + encryption::ViewingPublicKey, }; use log::info; use sequencer_service_rpc::RpcClient as _; @@ -70,8 +69,8 @@ async fn private_transfer_to_foreign_account() -> Result<()> { }); let result = wallet::cli::execute_subcommand(ctx.wallet_mut(), command).await?; - let SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash } = result else { - anyhow::bail!("Expected PrivacyPreservingTransfer return value"); + let SubcommandReturnValue::TransactionExecuted { tx_hash } = result else { + anyhow::bail!("Expected TransactionExecuted return value"); }; info!("Waiting for next block creation"); @@ -159,8 +158,8 @@ async fn private_transfer_to_owned_account_using_claiming_path() -> Result<()> { }); let sub_ret = wallet::cli::execute_subcommand(ctx.wallet_mut(), command).await?; - let SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash } = sub_ret else { - anyhow::bail!("Expected PrivacyPreservingTransfer return value"); + let SubcommandReturnValue::TransactionExecuted { tx_hash } = sub_ret else { + anyhow::bail!("Expected TransactionExecuted return value"); }; let tx = fetch_privacy_preserving_tx(ctx.sequencer_client(), tx_hash).await; @@ -238,8 +237,8 @@ async fn shielded_transfer_to_foreign_account() -> Result<()> { }); let result = wallet::cli::execute_subcommand(ctx.wallet_mut(), command).await?; - let SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash } = result else { - anyhow::bail!("Expected PrivacyPreservingTransfer return value"); + let SubcommandReturnValue::TransactionExecuted { tx_hash } = result else { + anyhow::bail!("Expected TransactionExecuted return value"); }; info!("Waiting for next block creation"); @@ -298,7 +297,7 @@ async fn private_transfer_to_owned_account_continuous_run_path() -> Result<()> { }); let sub_ret = wallet::cli::execute_subcommand(ctx.wallet_mut(), command).await?; - let SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash } = sub_ret else { + let SubcommandReturnValue::TransactionExecuted { tx_hash } = sub_ret else { anyhow::bail!("Failed to send transaction"); }; @@ -499,14 +498,14 @@ async fn shielded_transfers_to_two_identifiers_same_npk() -> Result<()> { sync_private(&mut ctx).await?; // Both accounts must be discovered with the correct balances. - let account_id_1 = AccountId::for_regular_private_account(&npk, identifier_1); + let account_id_1 = AccountId::for_regular_private_account(&npk, &vpk, identifier_1); let acc_1 = ctx .wallet() .get_account_private(account_id_1) .context("account for identifier 1 not found after sync")?; assert_eq!(acc_1.balance, 100); - let account_id_2 = AccountId::for_regular_private_account(&npk, identifier_2); + let account_id_2 = AccountId::for_regular_private_account(&npk, &vpk, identifier_2); let acc_2 = ctx .wallet() .get_account_private(account_id_2) @@ -562,11 +561,9 @@ async fn ppt_cant_chain_call_faucet() -> Result<()> { let nsk: lee_core::NullifierSecretKey = [3; 32]; let npk = NullifierPublicKey::from(&nsk); let vpk = ViewingPublicKey::from_bytes(vec![4_u8; 1184]).unwrap(); - let ssk = SharedSecretKey([55_u8; 32]); - let epk = EphemeralPublicKey(vec![55_u8; ML_KEM_768_CIPHERTEXT_LEN]); let attacker_vault_id = { let seed = vault_core::compute_vault_seed(attacker_id); - AccountId::for_private_pda(&vault_program_id, &seed, &npk, 1337) + AccountId::for_private_pda(&vault_program_id, &seed, &npk, &vpk, 1337) }; let amount: u128 = 1; @@ -600,10 +597,9 @@ async fn ppt_cant_chain_call_faucet() -> Result<()> { vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivatePdaInit { - epk, - view_tag: EncryptedAccountData::compute_view_tag(&npk, &vpk), + vpk, + random_seed: [0; 32], npk, - ssk, identifier: 1337, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -632,8 +628,7 @@ async fn prove_init_with_commitment_root( let nsk: lee_core::NullifierSecretKey = [7; 32]; let npk = NullifierPublicKey::from(&nsk); let vpk = ViewingPublicKey::from_bytes(vec![4_u8; 1184]).unwrap(); - let ssk = SharedSecretKey([55_u8; 32]); - let recipient_account_id = AccountId::for_regular_private_account(&npk, 0); + let recipient_account_id = AccountId::for_regular_private_account(&npk, &vpk, 0); let recipient = AccountWithMetadata::new(Account::default(), false, recipient_account_id); let (output, _) = execute_and_prove( @@ -644,10 +639,9 @@ async fn prove_init_with_commitment_root( vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &vpk), + vpk, + random_seed: [0; 32], npk, - ssk, identifier: 0, commitment_root, }, @@ -671,7 +665,8 @@ async fn init_with_dummy_commitment_root_produces_valid_root() -> Result<()> { let nsk: lee_core::NullifierSecretKey = [7; 32]; let npk = NullifierPublicKey::from(&nsk); - let recipient_account_id = AccountId::for_regular_private_account(&npk, 0); + let vpk = ViewingPublicKey::from_bytes(vec![4_u8; 1184]).unwrap(); + let recipient_account_id = AccountId::for_regular_private_account(&npk, &vpk, 0); let output = prove_init_with_commitment_root(&ctx, expected_digest).await?; diff --git a/integration_tests/tests/indexer_stall.rs b/integration_tests/tests/indexer_stall.rs new file mode 100644 index 00000000..ae1b8b6a --- /dev/null +++ b/integration_tests/tests/indexer_stall.rs @@ -0,0 +1,54 @@ +#![expect( + clippy::tests_outside_test_module, + reason = "We don't care about these in tests" +)] + +use std::time::Duration; + +use anyhow::{Context as _, Result}; +use indexer_service_protocol::IndexerSyncState; +use indexer_service_rpc::RpcClient as _; +use integration_tests::{TestContext, wait_for_indexer_to_catch_up}; +use log::info; + +const CAUGHT_UP_STATUS_TIMEOUT: Duration = Duration::from_secs(60); + +/// Test that the indexer status RPC reports caught-up with no stall after a clean run. +/// +/// The sequencer keeps producing blocks while we assert, so the status is polled until a +/// `CaughtUp` snapshot is observed and the indexed tip is checked as a lower bound. +/// +/// TODO: Integration-level park testing (publishing a bad block to force a stall) is a follow-up +/// needing fault injection support in the test harness. +#[tokio::test] +async fn indexer_status_rpc_reports_caught_up_with_no_stall() -> Result<()> { + let ctx = TestContext::new().await?; + + let indexer_tip = wait_for_indexer_to_catch_up(&ctx).await?; + + let status = tokio::time::timeout(CAUGHT_UP_STATUS_TIMEOUT, async { + loop { + let status = ctx.indexer_client().get_status().await?; + if status.state == IndexerSyncState::CaughtUp { + return anyhow::Ok(status); + } + info!("Waiting for caught-up indexer status, got {status:?}"); + tokio::time::sleep(Duration::from_millis(500)).await; + } + }) + .await + .context("Timed out waiting for indexer status to report caught-up")??; + + assert!( + status.stall_reason.is_none(), + "indexer should have no stall reason after a clean run, got {status:?}" + ); + // test for >= here because the sequencer keeps producing blocks while we assert, + // so the indexed tip may be ahead of the tip we observed when we waited for caught-up. + assert!( + status.indexed_block_id >= Some(indexer_tip), + "status indexed_block_id should be at least the caught-up tip {indexer_tip}, got {status:?}" + ); + + Ok(()) +} diff --git a/integration_tests/tests/keys.rs b/integration_tests/tests/keys.rs index 7fc73c62..10b5710e 100644 --- a/integration_tests/tests/keys.rs +++ b/integration_tests/tests/keys.rs @@ -57,8 +57,8 @@ async fn sync_private_account_with_non_zero_chain_index() -> Result<()> { }); let sub_ret = wallet::cli::execute_subcommand(ctx.wallet_mut(), command).await?; - let SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash } = sub_ret else { - anyhow::bail!("Expected PrivacyPreservingTransfer return value"); + let SubcommandReturnValue::TransactionExecuted { tx_hash } = sub_ret else { + anyhow::bail!("Expected TransactionExecuted return value"); }; let tx = fetch_privacy_preserving_tx(ctx.sequencer_client(), tx_hash).await; diff --git a/integration_tests/tests/pinata.rs b/integration_tests/tests/pinata.rs index 167d9b66..f2c634a2 100644 --- a/integration_tests/tests/pinata.rs +++ b/integration_tests/tests/pinata.rs @@ -162,8 +162,8 @@ async fn claim_pinata_to_existing_private_account() -> Result<()> { let pinata_balance_pre = account_balance(&ctx, system_accounts::pinata_account_id()).await?; let result = wallet::cli::execute_subcommand(ctx.wallet_mut(), command).await?; - let SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash: _ } = result else { - anyhow::bail!("Expected PrivacyPreservingTransfer return value"); + let SubcommandReturnValue::TransactionExecuted { tx_hash: _ } = result else { + anyhow::bail!("Expected TransactionExecuted return value"); }; info!("Waiting for next block creation"); diff --git a/integration_tests/tests/private_pda.rs b/integration_tests/tests/private_pda.rs index 2ccdfad8..0c67049f 100644 --- a/integration_tests/tests/private_pda.rs +++ b/integration_tests/tests/private_pda.rs @@ -11,7 +11,6 @@ use common::transaction::LeeTransaction; use integration_tests::{ TIME_TO_WAIT_FOR_BLOCK_SECONDS, TestContext, sync_private, verify_commitment_is_in_state, }; -use key_protocol::key_management::ephemeral_key_holder::EphemeralKeyHolder; use lee::{ AccountId, PrivacyPreservingTransaction, ProgramId, privacy_preserving_transaction::{ @@ -22,7 +21,7 @@ use lee::{ program::Program, }; use lee_core::{ - DUMMY_COMMITMENT_HASH, EncryptedAccountData, InputAccountIdentity, NullifierPublicKey, + DUMMY_COMMITMENT_HASH, InputAccountIdentity, NullifierPublicKey, account::{Account, AccountWithMetadata}, encryption::ViewingPublicKey, program::PdaSeed, @@ -48,7 +47,8 @@ async fn fund_private_pda( amount: u128, auth_transfer: &ProgramWithDependencies, ) -> Result<()> { - let pda_account_id = AccountId::for_private_pda(&authority_program_id, &seed, &npk, identifier); + let pda_account_id = + AccountId::for_private_pda(&authority_program_id, &seed, &npk, &vpk, identifier); let sender_account = wallet .get_account_public(sender) .await @@ -60,20 +60,15 @@ async fn fund_private_pda( let sender_pre = AccountWithMetadata::new(sender_account.clone(), true, sender); let pda_pre = AccountWithMetadata::new(Account::default(), false, pda_account_id); - let eph_holder = EphemeralKeyHolder::new(&vpk); - let ssk = eph_holder.calculate_shared_secret_sender(); - let epk = eph_holder.ephemeral_public_key().clone(); - let instruction = Program::serialize_instruction(AuthTransferInstruction::Transfer { amount }) .context("failed to serialize auth_transfer instruction")?; let account_identities = vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivatePdaInit { - epk, - view_tag: EncryptedAccountData::compute_view_tag(&npk, &vpk), + vpk, + random_seed: [0; 32], npk, - ssk, identifier, commitment_root: DUMMY_COMMITMENT_HASH, seed: Some((seed, authority_program_id)), @@ -173,8 +168,8 @@ async fn private_pda_family_members_receive_and_spend() -> Result<()> { let spend_program = ProgramWithDependencies::new(proxy, [(auth_transfer_id, auth_transfer)].into()); - let alice_pda_0_id = AccountId::for_private_pda(&proxy_id, &seed, &alice_npk, 0); - let alice_pda_1_id = AccountId::for_private_pda(&proxy_id, &seed, &alice_npk, 1); + let alice_pda_0_id = AccountId::for_private_pda(&proxy_id, &seed, &alice_npk, &alice_vpk, 0); + let alice_pda_1_id = AccountId::for_private_pda(&proxy_id, &seed, &alice_npk, &alice_vpk, 1); // Use two different public senders to avoid nonce conflicts between the back-to-back txs. let senders = ctx.existing_public_accounts(); diff --git a/integration_tests/tests/program_deployment.rs b/integration_tests/tests/program_deployment.rs index 2b242549..c92daeb3 100644 --- a/integration_tests/tests/program_deployment.rs +++ b/integration_tests/tests/program_deployment.rs @@ -11,7 +11,7 @@ use integration_tests::{TIME_TO_WAIT_FOR_BLOCK_SECONDS, TestContext, get_account use log::info; use sequencer_service_rpc::RpcClient as _; use tokio::test; -use wallet::cli::Command; +use wallet::{cli::Command, config::WalletConfigOverrides}; #[test] async fn deploy_and_execute_program() -> Result<()> { @@ -29,9 +29,6 @@ async fn deploy_and_execute_program() -> Result<()> { wallet::cli::execute_subcommand(ctx.wallet_mut(), command).await?; - info!("Waiting for next block creation"); - tokio::time::sleep(Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS)).await; - let account_id = new_account(&mut ctx, false, None).await?; let nonces = ctx.wallet().get_accounts_nonces(vec![account_id]).await?; @@ -65,3 +62,37 @@ async fn deploy_and_execute_program() -> Result<()> { Ok(()) } + +#[test] +async fn deploy_invalid_program_fails() -> Result<()> { + // An invalid program bytecode is rejected by the sequencer during block production, so the + // deployment transaction is never included in a block. Shrink the wallet's polling window so + // the command gives up quickly instead of waiting for the full default timeout. + let mut ctx = TestContext::builder() + .with_wallet_config_overrides(WalletConfigOverrides { + seq_poll_timeout: Some(Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS)), + seq_tx_poll_max_blocks: Some(5), + seq_poll_max_retries: Some(2), + ..WalletConfigOverrides::default() + }) + .build() + .await?; + + let mut tempfile = tempfile::NamedTempFile::new()?; + tempfile.write_all(b"this is not a valid program binary")?; + + let command = Command::DeployProgram { + binary_filepath: tempfile.path().to_owned(), + }; + + let result = wallet::cli::execute_subcommand(ctx.wallet_mut(), command).await; + + assert!( + result.is_err(), + "Deploying an invalid program should fail, but got: {result:?}" + ); + + info!("Deploying an invalid program failed as expected"); + + Ok(()) +} diff --git a/integration_tests/tests/tps.rs b/integration_tests/tests/tps.rs index 0b9a1002..71fb132b 100644 --- a/integration_tests/tests/tps.rs +++ b/integration_tests/tests/tps.rs @@ -15,7 +15,6 @@ use anyhow::{Context as _, Result}; use bytesize::ByteSize; use common::transaction::LeeTransaction; use integration_tests::{TestContext, config::SequencerPartialConfig}; -use key_protocol::key_management::ephemeral_key_holder::EphemeralKeyHolder; use lee::{ Account, AccountId, PrivacyPreservingTransaction, PrivateKey, PublicKey, PublicTransaction, privacy_preserving_transaction::{self as pptx, circuit}, @@ -23,8 +22,7 @@ use lee::{ public_transaction as putx, }; use lee_core::{ - DUMMY_COMMITMENT_HASH, EncryptedAccountData, InputAccountIdentity, MembershipProof, - NullifierPublicKey, + DUMMY_COMMITMENT_HASH, InputAccountIdentity, MembershipProof, NullifierPublicKey, account::{AccountWithMetadata, Nonce, data::Data}, encryption::ViewingPublicKey, }; @@ -267,7 +265,7 @@ fn build_privacy_transaction() -> PrivacyPreservingTransaction { data: Data::default(), }, true, - AccountId::for_regular_private_account(&sender_npk, 0), + AccountId::for_regular_private_account(&sender_npk, &sender_vpk, 0), ); let recipient_nsk = [2; 32]; let recipient_vpk = ViewingPublicKey::from_seed(&[101_u8; 32], &[102_u8; 32]); @@ -275,17 +273,9 @@ fn build_privacy_transaction() -> PrivacyPreservingTransaction { let recipient_pre = AccountWithMetadata::new( Account::default(), false, - AccountId::for_regular_private_account(&recipient_npk, 0), + AccountId::for_regular_private_account(&recipient_npk, &recipient_vpk, 0), ); - let eph_holder_from = EphemeralKeyHolder::new(&sender_vpk); - let sender_ss = eph_holder_from.calculate_shared_secret_sender(); - let sender_epk = eph_holder_from.ephemeral_public_key().clone(); - - let eph_holder_to = EphemeralKeyHolder::new(&recipient_vpk); - let recipient_ss = eph_holder_to.calculate_shared_secret_sender(); - let recipient_epk = eph_holder_to.ephemeral_public_key().clone(); - let balance_to_move: u128 = 1; let proof: MembershipProof = ( 1, @@ -302,18 +292,16 @@ fn build_privacy_transaction() -> PrivacyPreservingTransaction { .unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: sender_epk, - view_tag: EncryptedAccountData::compute_view_tag(&sender_npk, &sender_vpk), - ssk: sender_ss, + vpk: sender_vpk, + random_seed: [0; 32], nsk: sender_nsk, membership_proof: proof, identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: recipient_epk, - view_tag: EncryptedAccountData::compute_view_tag(&recipient_npk, &recipient_vpk), + vpk: recipient_vpk, + random_seed: [0; 32], npk: recipient_npk, - ssk: recipient_ss, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }, diff --git a/integration_tests/tests/vault.rs b/integration_tests/tests/vault.rs index e9ea2075..70be480e 100644 --- a/integration_tests/tests/vault.rs +++ b/integration_tests/tests/vault.rs @@ -30,7 +30,7 @@ async fn public_transfer_and_public_claim() -> Result<()> { .get_account_balance(recipient_vault_id) .await?; - let transfer_result = wallet::cli::execute_subcommand( + wallet::cli::execute_subcommand( ctx.wallet_mut(), Command::Vault(VaultSubcommand::Transfer { from: public_mention(sender), @@ -39,10 +39,6 @@ async fn public_transfer_and_public_claim() -> Result<()> { }), ) .await?; - assert!( - matches!(transfer_result, SubcommandReturnValue::Empty), - "Expected Empty return value for public vault transfer" - ); let sender_balance_after_transfer = ctx.sequencer_client().get_account_balance(sender).await?; let recipient_balance_after_transfer = ctx @@ -64,7 +60,7 @@ async fn public_transfer_and_public_claim() -> Result<()> { recipient_vault_balance_before + amount ); - let claim_result = wallet::cli::execute_subcommand( + wallet::cli::execute_subcommand( ctx.wallet_mut(), Command::Vault(VaultSubcommand::Claim { account_id: public_mention(recipient), @@ -72,10 +68,6 @@ async fn public_transfer_and_public_claim() -> Result<()> { }), ) .await?; - assert!( - matches!(claim_result, SubcommandReturnValue::Empty), - "Expected Empty return value for public vault claim" - ); let sender_balance_after_claim = ctx.sequencer_client().get_account_balance(sender).await?; let recipient_balance_after_claim = ctx @@ -138,9 +130,9 @@ async fn private_transfer_and_private_claim() -> Result<()> { assert!( matches!( transfer_result, - SubcommandReturnValue::PrivacyPreservingTransfer { .. } + SubcommandReturnValue::TransactionExecuted { .. } ), - "Expected PrivacyPreservingTransfer return value for private vault transfer" + "Expected TransactionExecuted return value for private vault transfer" ); let sender_balance_after_transfer = ctx @@ -179,9 +171,9 @@ async fn private_transfer_and_private_claim() -> Result<()> { assert!( matches!( claim_result, - SubcommandReturnValue::PrivacyPreservingTransfer { .. } + SubcommandReturnValue::TransactionExecuted { .. } ), - "Expected PrivacyPreservingTransfer return value for private vault claim" + "Expected TransactionExecuted return value for private vault claim" ); let sender_balance_after_claim = ctx diff --git a/integration_tests/tests/wallet_ffi.rs b/integration_tests/tests/wallet_ffi.rs index 57f1afa4..321c874d 100644 --- a/integration_tests/tests/wallet_ffi.rs +++ b/integration_tests/tests/wallet_ffi.rs @@ -1012,7 +1012,11 @@ fn test_wallet_ffi_transfer_shielded() -> Result<()> { let (to, to_keys) = unsafe { let mut out_keys = FfiPrivateAccountKeys::default(); wallet_ffi_create_private_accounts_key(wallet_ffi_handle, &raw mut out_keys).unwrap(); - let account_id = lee::AccountId::for_regular_private_account(&out_keys.npk(), 0_u128); + let account_id = lee::AccountId::for_regular_private_account( + &out_keys.npk(), + &out_keys.vpk().unwrap(), + 0_u128, + ); let to: FfiBytes32 = account_id.into(); (to, out_keys) }; @@ -1155,7 +1159,11 @@ fn test_wallet_ffi_transfer_private() -> Result<()> { let (to, to_keys) = unsafe { let mut out_keys = FfiPrivateAccountKeys::default(); wallet_ffi_create_private_accounts_key(wallet_ffi_handle, &raw mut out_keys).unwrap(); - let account_id = lee::AccountId::for_regular_private_account(&out_keys.npk(), 0_u128); + let account_id = lee::AccountId::for_regular_private_account( + &out_keys.npk(), + &out_keys.vpk().unwrap(), + 0_u128, + ); let to: FfiBytes32 = account_id.into(); (to, out_keys) }; @@ -1236,7 +1244,11 @@ fn restore_keys_from_seed_ffi() -> Result<()> { let (private_account_id_1, private_account_1_keys) = unsafe { let mut out_keys = FfiPrivateAccountKeys::default(); wallet_ffi_create_private_accounts_key(wallet_ffi_handle, &raw mut out_keys).unwrap(); - let account_id = lee::AccountId::for_regular_private_account(&out_keys.npk(), 0_u128); + let account_id = lee::AccountId::for_regular_private_account( + &out_keys.npk(), + &out_keys.vpk().unwrap(), + 0_u128, + ); let to: FfiBytes32 = account_id.into(); (to, out_keys) }; @@ -1244,7 +1256,11 @@ fn restore_keys_from_seed_ffi() -> Result<()> { let (private_account_id_2, private_account_2_keys) = unsafe { let mut out_keys = FfiPrivateAccountKeys::default(); wallet_ffi_create_private_accounts_key(wallet_ffi_handle, &raw mut out_keys).unwrap(); - let account_id = lee::AccountId::for_regular_private_account(&out_keys.npk(), 0_u128); + let account_id = lee::AccountId::for_regular_private_account( + &out_keys.npk(), + &out_keys.vpk().unwrap(), + 0_u128, + ); let to: FfiBytes32 = account_id.into(); (to, out_keys) }; diff --git a/lee/key_protocol/src/key_management/group_key_holder.rs b/lee/key_protocol/src/key_management/group_key_holder.rs index 2fc6b5fb..0821e4cd 100644 --- a/lee/key_protocol/src/key_management/group_key_holder.rs +++ b/lee/key_protocol/src/key_management/group_key_holder.rs @@ -334,10 +334,10 @@ mod tests { let program_id: ProgramId = [9; 8]; let holder = GroupKeyHolder::from_gms(gms); - let npk = holder - .derive_keys_for_pda(&TEST_PROGRAM_ID, &seed) - .generate_nullifier_public_key(); - let account_id = AccountId::for_private_pda(&program_id, &seed, &npk, u128::MAX); + let keys = holder.derive_keys_for_pda(&TEST_PROGRAM_ID, &seed); + let npk = keys.generate_nullifier_public_key(); + let vpk = keys.generate_viewing_public_key(); + let account_id = AccountId::for_private_pda(&program_id, &seed, &npk, &vpk, u128::MAX); let expected_npk = NullifierPublicKey([ 136, 176, 234, 71, 208, 8, 143, 142, 126, 155, 132, 18, 71, 27, 88, 56, 100, 90, 79, @@ -346,7 +346,7 @@ mod tests { // AccountId is derived from (program_id, seed, npk), so it changes when npk changes. // We verify npk is pinned, and AccountId is deterministically derived from it. let expected_account_id = - AccountId::for_private_pda(&program_id, &seed, &expected_npk, u128::MAX); + AccountId::for_private_pda(&program_id, &seed, &expected_npk, &vpk, u128::MAX); assert_eq!(npk, expected_npk); assert_eq!(account_id, expected_account_id); @@ -543,13 +543,16 @@ mod tests { let bob_holder = GroupKeyHolder::unseal(&sealed, &bob_vsk).expect("Bob should unseal the GMS"); - let bob_npk = bob_holder - .derive_keys_for_pda(&TEST_PROGRAM_ID, &pda_seed) - .generate_nullifier_public_key(); + let bob_group_keys = bob_holder.derive_keys_for_pda(&TEST_PROGRAM_ID, &pda_seed); + let bob_npk = bob_group_keys.generate_nullifier_public_key(); assert_eq!(alice_npk, bob_npk); - let alice_account_id = AccountId::for_private_pda(&program_id, &pda_seed, &alice_npk, 0); - let bob_account_id = AccountId::for_private_pda(&program_id, &pda_seed, &bob_npk, 0); + let alice_vpk = alice_keys.generate_viewing_public_key(); + let bob_group_vpk = bob_group_keys.generate_viewing_public_key(); + let alice_account_id = + AccountId::for_private_pda(&program_id, &pda_seed, &alice_npk, &alice_vpk, 0); + let bob_account_id = + AccountId::for_private_pda(&program_id, &pda_seed, &bob_npk, &bob_group_vpk, 0); assert_eq!(alice_account_id, bob_account_id); } diff --git a/lee/key_protocol/src/key_management/key_tree/keys_private.rs b/lee/key_protocol/src/key_management/key_tree/keys_private.rs index 48b47975..8165e808 100644 --- a/lee/key_protocol/src/key_management/key_tree/keys_private.rs +++ b/lee/key_protocol/src/key_management/key_tree/keys_private.rs @@ -94,10 +94,11 @@ impl KeyTreeNode for ChildKeysPrivate { fn account_ids(&self) -> impl Iterator { let npk = self.value.0.nullifier_public_key; + let vpk = self.value.0.viewing_public_key.clone(); self.value .1 .keys() - .map(move |kind| lee::AccountId::for_private_account(&npk, kind)) + .map(move |kind| lee::AccountId::for_private_account(&npk, &vpk, kind)) } } diff --git a/lee/key_protocol/src/key_management/key_tree/mod.rs b/lee/key_protocol/src/key_management/key_tree/mod.rs index 7e4b432c..463c757a 100644 --- a/lee/key_protocol/src/key_management/key_tree/mod.rs +++ b/lee/key_protocol/src/key_management/key_tree/mod.rs @@ -268,6 +268,7 @@ impl KeyTree { let node = self.key_map.get(cci)?; let account_id = lee::AccountId::for_regular_private_account( &node.value.0.nullifier_public_key, + &node.value.0.viewing_public_key, identifier, ); if self.account_id_map.contains_key(&account_id) { diff --git a/lee/privacy_preserving_circuit/src/execution_state.rs b/lee/privacy_preserving_circuit/src/execution_state.rs index 4e81f9ce..09ad30af 100644 --- a/lee/privacy_preserving_circuit/src/execution_state.rs +++ b/lee/privacy_preserving_circuit/src/execution_state.rs @@ -6,6 +6,7 @@ use std::{ use lee_core::{ Identifier, InputAccountIdentity, NullifierPublicKey, account::{Account, AccountId, AccountWithMetadata}, + encryption::ViewingPublicKey, program::{ AccountPostState, BlockValidityWindow, ChainedCall, Claim, DEFAULT_PROGRAM_ID, MAX_NUMBER_CHAINED_CALLS, PdaSeed, ProgramId, ProgramOutput, TimestampValidityWindow, @@ -21,7 +22,7 @@ pub struct ExecutionState { block_validity_window: BlockValidityWindow, timestamp_validity_window: TimestampValidityWindow, /// Positions (in `pre_states`) of private-PDA accounts whose supplied npk has been bound to - /// their `AccountId` via a proven `AccountId::for_private_pda(program_id, seed, npk, + /// their `AccountId` via a proven `AccountId::for_private_pda(program_id, seed, npk, vpk, /// identifier)` check. /// Two proof paths populate this set: a `Claim::Pda(seed)` in a program's `post_state` on /// that `pre_state`, or a caller's `ChainedCall.pda_seeds` entry matching that `pre_state` @@ -43,12 +44,13 @@ pub struct ExecutionState { /// `AccountId` entry or as an equality check against the existing one, making the rule: one /// `(program, seed)` โ†’ one account per tx. pda_family_binding: HashMap<(ProgramId, PdaSeed), AccountId>, - /// Map from a private-PDA `pre_state`'s position in `account_identities` to the (npk, + /// Map from a private-PDA `pre_state`'s position in `account_identities` to the (npk, vpk, /// identifier) supplied for that position. Built once in `derive_from_outputs` by walking - /// `account_identities` and consulting `npk_if_private_pda`. Used later by the claim and + /// `account_identities` and consulting `npk_vpk_if_private_pda`. Used later by the claim and /// caller-seeds authorization paths to verify - /// `AccountId::for_private_pda(program_id, seed, npk, identifier) == pre_state.account_id`. - private_pda_npk_by_position: HashMap, + /// `AccountId::for_private_pda(program_id, seed, npk, vpk, identifier) == + /// pre_state.account_id`. + private_pda_by_position: HashMap, authorized_accounts: HashSet, } @@ -59,9 +61,48 @@ impl ExecutionState { program_id: ProgramId, program_outputs: Vec, ) -> Self { - let private_pda_npk_by_position = build_private_pda_npk_map(account_identities); - let (block_validity_window, timestamp_validity_window) = - intersect_validity_windows(&program_outputs); + // Build position โ†’ (npk, identifier) map for private-PDA pre_states, indexed by position + // in `account_identities`. The vec is documented as 1:1 with the program's pre_state + // order, so position here matches `pre_state_position` used downstream in + // `validate_and_sync_states`. + let mut private_pda_by_position: HashMap< + usize, + (NullifierPublicKey, ViewingPublicKey, Identifier), + > = HashMap::new(); + for (pos, account_identity) in account_identities.iter().enumerate() { + if let Some((npk, vpk, identifier)) = account_identity.npk_vpk_if_private_pda() { + private_pda_by_position.insert(pos, (npk, vpk, identifier)); + } + } + + let block_valid_from = program_outputs + .iter() + .filter_map(|output| output.block_validity_window.start()) + .max(); + let block_valid_until = program_outputs + .iter() + .filter_map(|output| output.block_validity_window.end()) + .min(); + let ts_valid_from = program_outputs + .iter() + .filter_map(|output| output.timestamp_validity_window.start()) + .max(); + let ts_valid_until = program_outputs + .iter() + .filter_map(|output| output.timestamp_validity_window.end()) + .min(); + + let block_validity_window: BlockValidityWindow = (block_valid_from, block_valid_until) + .try_into() + .expect( + "There should be non empty intersection in the program output block validity windows", + ); + let timestamp_validity_window: TimestampValidityWindow = + (ts_valid_from, ts_valid_until) + .try_into() + .expect( + "There should be non empty intersection in the program output timestamp validity windows", + ); let mut execution_state = Self { pre_states: Vec::new(), @@ -70,7 +111,7 @@ impl ExecutionState { timestamp_validity_window, private_pda_bound_positions: HashMap::new(), pda_family_binding: HashMap::new(), - private_pda_npk_by_position, + private_pda_by_position, authorized_accounts: HashSet::new(), }; @@ -99,7 +140,49 @@ impl ExecutionState { panic!("Insufficient program outputs for chained calls"); }; - verify_program_output(&chained_call, caller_program_id, &program_output); + // Check that instruction data in chained call is the instruction data in program output + assert_eq!( + chained_call.instruction_data, program_output.instruction_data, + "Mismatched instruction data between chained call and program output" + ); + + // Check that `program_output` is consistent with the execution of the corresponding + // program. + let program_output_words = + &to_vec(&program_output).expect("program_output must be serializable"); + env::verify(chained_call.program_id, program_output_words).unwrap_or_else( + |_: Infallible| unreachable!("Infallible error is never constructed"), + ); + + // Verify that the program output's self_program_id matches the expected program ID. + // This ensures the proof commits to which program produced the output. + assert_eq!( + program_output.self_program_id, chained_call.program_id, + "Program output self_program_id does not match chained call program_id" + ); + + // Verify that the program output's caller_program_id matches the actual caller. + // This prevents a malicious user from privately executing an internal function + // by spoofing caller_program_id (e.g. passing caller_program_id = self_program_id + // to bypass access control checks). + assert_eq!( + program_output.caller_program_id, caller_program_id, + "Program output caller_program_id does not match actual caller" + ); + + // Check that the program is well behaved. + // See the # Programs section for the definition of the `validate_execution` method. + let validated_execution = validate_execution( + &program_output.pre_states, + &program_output.post_states, + chained_call.program_id, + ); + if let Err(err) = validated_execution { + panic!( + "Invalid program behavior in program {:?}: {err}", + chained_call.program_id + ); + } for next_call in program_output.chained_calls.iter().rev() { chained_calls.push_front((next_call.clone(), Some(chained_call.program_id))); @@ -123,8 +206,41 @@ impl ExecutionState { "Inner call without a chained call found", ); - execution_state.assert_all_pda_positions_bound(account_identities); - execution_state.assert_modified_accounts_claimed(); + // Every private-PDA pre_state must have had its npk bound to its account_id, either via + // a `Claim::Pda(seed)` in some program's post_state or via a caller's `pda_seeds` + // matching the private derivation. An unbound private-PDA pre_state has no + // cryptographic link between the supplied npk and the account_id, and must be rejected. + for (pos, account_identity) in account_identities.iter().enumerate() { + if account_identity.is_private_pda() { + assert!( + execution_state + .private_pda_bound_positions + .contains_key(&pos), + "private PDA pre_state at position {pos} has no proven (seed, npk) binding via Claim::Pda or caller pda_seeds" + ); + } + } + + // Check that all modified uninitialized accounts were claimed + for (account_id, post) in execution_state + .pre_states + .iter() + .filter(|a| a.account.program_owner == DEFAULT_PROGRAM_ID) + .map(|a| { + let post = execution_state + .post_states + .get(&a.account_id) + .expect("Post state must exist for pre state"); + (a, post) + }) + .filter(|(pre_default, post)| pre_default.account != **post) + .map(|(pre, post)| (pre.account_id, post)) + { + assert_ne!( + post.program_owner, DEFAULT_PROGRAM_ID, + "Account {account_id} was modified but not claimed" + ); + } execution_state } @@ -142,185 +258,209 @@ impl ExecutionState { for (pre, mut post) in output_pre_states.into_iter().zip(output_post_states) { let pre_account_id = pre.account_id; let pre_is_authorized = pre.is_authorized; + let post_states_entry = self.post_states.entry(pre.account_id); + match &post_states_entry { + Entry::Occupied(occupied) => { + #[expect( + clippy::shadow_unrelated, + reason = "Shadowing is intentional to use all fields" + )] + let AccountWithMetadata { + account: pre_account, + account_id: pre_account_id, + is_authorized: pre_is_authorized, + } = pre; - if let Some(existing) = self.post_states.get(&pre.account_id) { - #[expect( - clippy::shadow_unrelated, - reason = "Shadowing is intentional to use all fields" - )] - let AccountWithMetadata { - account: pre_account, - account_id: pre_account_id, - is_authorized: pre_is_authorized, - } = pre; - - assert_eq!( - existing, &pre_account, - "Inconsistent pre state for account {pre_account_id}", - ); - - let (previous_is_authorized, pre_state_position) = self - .pre_states - .iter() - .enumerate() - .find(|(_, acc)| acc.account_id == pre_account_id) - .map_or_else( - || panic!( - "Pre state must exist in execution state for account {pre_account_id}", - ), - |(pos, acc)| (acc.is_authorized, pos), + // Ensure that new pre state is the same as known post state + assert_eq!( + occupied.get(), + &pre_account, + "Inconsistent pre state for account {pre_account_id}", ); - let is_authorized = resolve_authorization_and_record_bindings( - &mut self.pda_family_binding, - &mut self.private_pda_bound_positions, - &self.private_pda_npk_by_position, - &mut self.authorized_accounts, - pre_account_id, - pre_state_position, - caller_program_id, - caller_pda_seeds, - previous_is_authorized, - ); + let (previous_is_authorized, pre_state_position) = self + .pre_states + .iter() + .enumerate() + .find(|(_, acc)| acc.account_id == pre_account_id) + .map_or_else( + || panic!( + "Pre state must exist in execution state for account {pre_account_id}", + ), + |(pos, acc)| (acc.is_authorized, pos) + ); - assert_eq!( - pre_is_authorized, is_authorized, - "Inconsistent authorization for account {pre_account_id}", - ); - } else { - let pre_state_position = self.pre_states.len(); - resolve_external_seed( - account_identities, - pre_state_position, - pre_account_id, - pre.is_authorized, - &mut self.private_pda_bound_positions, - &mut self.pda_family_binding, - ); - self.pre_states.push(pre); + let is_authorized = resolve_authorization_and_record_bindings( + &mut self.pda_family_binding, + &mut self.private_pda_bound_positions, + &self.private_pda_by_position, + &mut self.authorized_accounts, + pre_account_id, + pre_state_position, + caller_program_id, + caller_pda_seeds, + previous_is_authorized, + ); + + assert_eq!( + pre_is_authorized, is_authorized, + "Inconsistent authorization for account {pre_account_id}", + ); + } + Entry::Vacant(_) => { + // Pre state for the initial call + let pre_state_position = self.pre_states.len(); + let external_seed = match account_identities.get(pre_state_position) { + Some(InputAccountIdentity::PrivatePdaInit { + npk, + vpk, + identifier, + seed: Some((seed, authority_program_id)), + .. + }) => { + let expected = AccountId::for_private_pda( + authority_program_id, + seed, + npk, + vpk, + *identifier, + ); + assert_eq!( + pre_account_id, expected, + "External seed mismatch for PrivatePdaInit at position {pre_state_position}" + ); + Some((*seed, *authority_program_id)) + } + Some(InputAccountIdentity::PrivatePdaUpdate { + nsk, + vpk, + identifier, + seed: Some((seed, authority_program_id)), + .. + }) => { + let npk = NullifierPublicKey::from(nsk); + let expected = AccountId::for_private_pda( + authority_program_id, + seed, + &npk, + vpk, + *identifier, + ); + assert_eq!( + pre_account_id, expected, + "External seed mismatch for PrivatePdaUpdate at position {pre_state_position}" + ); + Some((*seed, *authority_program_id)) + } + _ => None, + }; + // External seed is only consulted the first time the account is seen. + // Subsequent calls need no re-check because the entry is already recorded on + // private_pda_bound_positions. + if let Some((seed, authority_program_id)) = external_seed { + assert!( + !pre.is_authorized, + "Private PDA with externally-provided seed must not be authorized at position {pre_state_position}" + ); + bind_private_pda_position( + &mut self.private_pda_bound_positions, + pre_state_position, + authority_program_id, + seed, + ); + assert_family_binding( + &mut self.pda_family_binding, + authority_program_id, + seed, + pre_account_id, + ); + } + self.pre_states.push(pre); + } } if let Some(claim) = post.required_claim() { - self.process_claim( - account_identities, - &mut post, - pre_account_id, - pre_is_authorized, - program_id, - claim, + // The invoked program can only claim accounts with default program id. + assert_eq!( + post.account().program_owner, + DEFAULT_PROGRAM_ID, + "Cannot claim an initialized account {pre_account_id}" ); - } - self.post_states.insert(pre_account_id, post.into_account()); - } - } + let pre_state_position = self + .pre_states + .iter() + .position(|acc| acc.account_id == pre_account_id) + .expect("Pre state must exist at this point"); - fn process_claim( - &mut self, - account_identities: &[InputAccountIdentity], - post: &mut AccountPostState, - pre_account_id: AccountId, - pre_is_authorized: bool, - program_id: ProgramId, - claim: Claim, - ) { - assert_eq!( - post.account().program_owner, - DEFAULT_PROGRAM_ID, - "Cannot claim an initialized account {pre_account_id}" - ); - - let pre_state_position = self - .pre_states - .iter() - .position(|acc| acc.account_id == pre_account_id) - .expect("Pre state must exist at this point"); - - let account_identity = &account_identities[pre_state_position]; - if account_identity.is_public() { - match claim { - Claim::Authorized => { - assert!( - pre_is_authorized, - "Cannot claim unauthorized account {pre_account_id}" - ); + let account_identity = &account_identities[pre_state_position]; + if account_identity.is_public() { + match claim { + Claim::Authorized => { + // Note: no need to check authorized pdas because we have already + // checked consistency of authorization above. + assert!( + pre_is_authorized, + "Cannot claim unauthorized account {pre_account_id}" + ); + } + Claim::Pda(seed) => { + let pda = AccountId::for_public_pda(&program_id, &seed); + assert_eq!( + pre_account_id, pda, + "Invalid PDA claim for account {pre_account_id} which does not match derived PDA {pda}" + ); + assert_family_binding( + &mut self.pda_family_binding, + program_id, + seed, + pre_account_id, + ); + } + } + } else { + // Private accounts: don't enforce the claim semantics. Unauthorized private + // claiming is intentionally allowed + match claim { + Claim::Authorized => {} + Claim::Pda(seed) => { + let (npk, vpk, identifier) = self + .private_pda_by_position + .get(&pre_state_position) + .expect( + "private PDA pre_state must have an npk in the position map", + ); + let pda = AccountId::for_private_pda( + &program_id, + &seed, + npk, + vpk, + *identifier, + ); + assert_eq!( + pre_account_id, pda, + "Invalid private PDA claim for account {pre_account_id}" + ); + bind_private_pda_position( + &mut self.private_pda_bound_positions, + pre_state_position, + program_id, + seed, + ); + assert_family_binding( + &mut self.pda_family_binding, + program_id, + seed, + pre_account_id, + ); + } + } } - Claim::Pda(seed) => { - let pda = AccountId::for_public_pda(&program_id, &seed); - assert_eq!( - pre_account_id, pda, - "Invalid PDA claim for account {pre_account_id} which does not match derived PDA {pda}" - ); - assert_family_binding( - &mut self.pda_family_binding, - program_id, - seed, - pre_account_id, - ); - } - } - } else { - match claim { - Claim::Authorized => {} - Claim::Pda(seed) => { - let (npk, identifier) = self - .private_pda_npk_by_position - .get(&pre_state_position) - .expect("private PDA pre_state must have an npk in the position map"); - let pda = AccountId::for_private_pda(&program_id, &seed, npk, *identifier); - assert_eq!( - pre_account_id, pda, - "Invalid private PDA claim for account {pre_account_id}" - ); - bind_private_pda_position( - &mut self.private_pda_bound_positions, - pre_state_position, - program_id, - seed, - ); - assert_family_binding( - &mut self.pda_family_binding, - program_id, - seed, - pre_account_id, - ); - } - } - } - post.account_mut().program_owner = program_id; - } - - fn assert_all_pda_positions_bound(&self, account_identities: &[InputAccountIdentity]) { - for (pos, account_identity) in account_identities.iter().enumerate() { - if account_identity.is_private_pda() { - assert!( - self.private_pda_bound_positions.contains_key(&pos), - "private PDA pre_state at position {pos} has no proven (seed, npk) binding via Claim::Pda or caller pda_seeds" - ); + post.account_mut().program_owner = program_id; } - } - } - fn assert_modified_accounts_claimed(&self) { - for (account_id, post) in self - .pre_states - .iter() - .filter(|a| a.account.program_owner == DEFAULT_PROGRAM_ID) - .map(|a| { - let post = self - .post_states - .get(&a.account_id) - .expect("Post state must exist for pre state"); - (a, post) - }) - .filter(|(pre_default, post)| pre_default.account != **post) - .map(|(pre, post)| (pre.account_id, post)) - { - assert_ne!( - post.program_owner, DEFAULT_PROGRAM_ID, - "Account {account_id} was modified but not claimed" - ); + post_states_entry.insert_entry(post.into_account()); } } @@ -359,150 +499,6 @@ impl ExecutionState { } } -fn verify_program_output( - chained_call: &ChainedCall, - caller_program_id: Option, - program_output: &ProgramOutput, -) { - assert_eq!( - chained_call.instruction_data, program_output.instruction_data, - "Mismatched instruction data between chained call and program output" - ); - - let program_output_words = - &to_vec(program_output).expect("program_output must be serializable"); - env::verify(chained_call.program_id, program_output_words) - .unwrap_or_else(|_: Infallible| unreachable!("Infallible error is never constructed")); - - assert_eq!( - program_output.self_program_id, chained_call.program_id, - "Program output self_program_id does not match chained call program_id" - ); - - assert_eq!( - program_output.caller_program_id, caller_program_id, - "Program output caller_program_id does not match actual caller" - ); - - if let Err(err) = validate_execution( - &program_output.pre_states, - &program_output.post_states, - chained_call.program_id, - ) { - panic!( - "Invalid program behavior in program {:?}: {err}", - chained_call.program_id - ); - } -} - -fn build_private_pda_npk_map( - account_identities: &[InputAccountIdentity], -) -> HashMap { - account_identities - .iter() - .enumerate() - .filter_map(|(pos, identity)| { - identity - .npk_if_private_pda() - .map(|(npk, identifier)| (pos, (npk, identifier))) - }) - .collect() -} - -fn intersect_validity_windows( - program_outputs: &[ProgramOutput], -) -> (BlockValidityWindow, TimestampValidityWindow) { - let block_valid_from = program_outputs - .iter() - .filter_map(|output| output.block_validity_window.start()) - .max(); - let block_valid_until = program_outputs - .iter() - .filter_map(|output| output.block_validity_window.end()) - .min(); - let ts_valid_from = program_outputs - .iter() - .filter_map(|output| output.timestamp_validity_window.start()) - .max(); - let ts_valid_until = program_outputs - .iter() - .filter_map(|output| output.timestamp_validity_window.end()) - .min(); - - let block_validity_window: BlockValidityWindow = - (block_valid_from, block_valid_until).try_into().expect( - "There should be non empty intersection in the program output block validity windows", - ); - let timestamp_validity_window: TimestampValidityWindow = (ts_valid_from, ts_valid_until) - .try_into() - .expect( - "There should be non empty intersection in the program output timestamp validity windows", - ); - - (block_validity_window, timestamp_validity_window) -} - -fn resolve_external_seed( - account_identities: &[InputAccountIdentity], - pre_state_position: usize, - pre_account_id: AccountId, - is_authorized: bool, - private_pda_bound_positions: &mut HashMap, - pda_family_binding: &mut HashMap<(ProgramId, PdaSeed), AccountId>, -) { - let external_seed = match account_identities.get(pre_state_position) { - Some(InputAccountIdentity::PrivatePdaInit { - npk, - identifier, - seed: Some((seed, authority_program_id)), - .. - }) => { - let expected = AccountId::for_private_pda(authority_program_id, seed, npk, *identifier); - assert_eq!( - pre_account_id, expected, - "External seed mismatch for PrivatePdaInit at position {pre_state_position}" - ); - Some((*seed, *authority_program_id)) - } - Some(InputAccountIdentity::PrivatePdaUpdate { - nsk, - identifier, - seed: Some((seed, authority_program_id)), - .. - }) => { - let npk = NullifierPublicKey::from(nsk); - let expected = - AccountId::for_private_pda(authority_program_id, seed, &npk, *identifier); - assert_eq!( - pre_account_id, expected, - "External seed mismatch for PrivatePdaUpdate at position {pre_state_position}" - ); - Some((*seed, *authority_program_id)) - } - _ => None, - }; - - if let Some((seed, authority_program_id)) = external_seed { - assert!( - !is_authorized, - "Private PDA with externally-provided seed must not be authorized at position {pre_state_position}" - ); - bind_private_pda_position( - private_pda_bound_positions, - pre_state_position, - authority_program_id, - seed, - ); - assert_family_binding( - pda_family_binding, - authority_program_id, - seed, - pre_account_id, - ); - } -} - /// Record or re-verify the `(program_id, seed) โ†’ account_id` family binding for the /// transaction. Any claim or caller-seed authorization that resolves a `pre_state` under /// `(program_id, seed)` must agree with every prior resolution of the same pair; otherwise a @@ -565,7 +561,7 @@ fn bind_private_pda_position( fn resolve_authorization_and_record_bindings( pda_family_binding: &mut HashMap<(ProgramId, PdaSeed), AccountId>, private_pda_bound_positions: &mut HashMap, - private_pda_npk_by_position: &HashMap, + private_pda_by_position: &HashMap, authorized_accounts: &mut HashSet, pre_account_id: AccountId, pre_state_position: usize, @@ -579,9 +575,10 @@ fn resolve_authorization_and_record_bindings( if AccountId::for_public_pda(&caller, seed) == pre_account_id { return Some((*seed, false, caller)); } - if let Some((npk, identifier)) = - private_pda_npk_by_position.get(&pre_state_position) - && AccountId::for_private_pda(&caller, seed, npk, *identifier) == pre_account_id + if let Some((npk, vpk, identifier)) = + private_pda_by_position.get(&pre_state_position) + && AccountId::for_private_pda(&caller, seed, npk, vpk, *identifier) + == pre_account_id { return Some((*seed, true, caller)); } diff --git a/lee/privacy_preserving_circuit/src/output.rs b/lee/privacy_preserving_circuit/src/output.rs index 74c1cccc..c9543e05 100644 --- a/lee/privacy_preserving_circuit/src/output.rs +++ b/lee/privacy_preserving_circuit/src/output.rs @@ -1,244 +1,14 @@ use lee_core::{ - Commitment, CommitmentSetDigest, EncryptedAccountData, EncryptionScheme, EphemeralPublicKey, + Commitment, CommitmentSetDigest, EncryptedAccountData, EncryptionScheme, EphemeralSecretKey, InputAccountIdentity, MembershipProof, Nullifier, NullifierPublicKey, NullifierSecretKey, PrivacyPreservingCircuitOutput, PrivateAccountKind, SharedSecretKey, account::{Account, AccountId, Nonce}, compute_digest_for_path, + encryption::ViewingPublicKey, }; use crate::execution_state::ExecutionState; -struct PrivateOutputHandler<'ctx> { - output: &'ctx mut PrivacyPreservingCircuitOutput, - output_index: &'ctx mut u32, - pre_state: &'ctx lee_core::account::AccountWithMetadata, - post_state: Account, - epk: &'ctx EphemeralPublicKey, - view_tag: u8, - ssk: &'ctx SharedSecretKey, - identifier: u128, -} - -impl PrivateOutputHandler<'_> { - fn authorized_init(self, nsk: &NullifierSecretKey, commitment_root: &CommitmentSetDigest) { - let npk = NullifierPublicKey::from(nsk); - let account_id = - derive_and_verify_account_id(&npk, self.identifier, self.pre_state.account_id); - - assert!( - self.pre_state.is_authorized, - "Pre-state not authorized for authenticated private account" - ); - assert_eq!( - self.pre_state.account, - Account::default(), - "Found new private account with non default values" - ); - - let (new_nullifier, new_nonce) = init_nullifier_and_nonce(&account_id, commitment_root); - let kind = PrivateAccountKind::Regular(self.identifier); - - self.emit_private_output(&account_id, &kind, new_nullifier, new_nonce); - } - - fn authorized_update(self, nsk: &NullifierSecretKey, membership_proof: &MembershipProof) { - let npk = NullifierPublicKey::from(nsk); - let account_id = - derive_and_verify_account_id(&npk, self.identifier, self.pre_state.account_id); - - assert!( - self.pre_state.is_authorized, - "Pre-state not authorized for authenticated private account" - ); - - let new_nullifier = compute_update_nullifier_and_set_digest( - membership_proof, - &self.pre_state.account, - &account_id, - nsk, - ); - let new_nonce = self - .pre_state - .account - .nonce - .private_account_nonce_increment(nsk); - let kind = PrivateAccountKind::Regular(self.identifier); - - self.emit_private_output(&account_id, &kind, new_nullifier, new_nonce); - } - - fn unauthorized(self, npk: &NullifierPublicKey, commitment_root: &CommitmentSetDigest) { - let account_id = - derive_and_verify_account_id(npk, self.identifier, self.pre_state.account_id); - - assert_eq!( - self.pre_state.account, - Account::default(), - "Found new private account with non default values", - ); - assert!( - !self.pre_state.is_authorized, - "Found new private account marked as authorized." - ); - - let (new_nullifier, new_nonce) = init_nullifier_and_nonce(&account_id, commitment_root); - let kind = PrivateAccountKind::Regular(self.identifier); - - self.emit_private_output(&account_id, &kind, new_nullifier, new_nonce); - } - - fn pda_init( - self, - commitment_root: &CommitmentSetDigest, - pos: usize, - pda_seed_by_position: &std::collections::HashMap< - usize, - (lee_core::program::ProgramId, lee_core::program::PdaSeed), - >, - ) { - // The npk-to-account_id binding is established upstream in - // `validate_and_sync_states` via `Claim::Pda(seed)` or a caller `pda_seeds` - // match. Here we only enforce the init pre-conditions. - assert!( - !self.pre_state.is_authorized, - "PrivatePdaInit requires unauthorized pre_state" - ); - assert_eq!( - self.pre_state.account, - Account::default(), - "New private PDA must be default" - ); - - let (new_nullifier, new_nonce) = - init_nullifier_and_nonce(&self.pre_state.account_id, commitment_root); - - let account_id = self.pre_state.account_id; - let (authority_program_id, seed) = pda_seed_by_position - .get(&pos) - .expect("PrivatePdaInit position must be in pda_seed_by_position"); - let kind = PrivateAccountKind::Pda { - program_id: *authority_program_id, - seed: *seed, - identifier: self.identifier, - }; - - self.emit_private_output(&account_id, &kind, new_nullifier, new_nonce); - } - - fn pda_update( - self, - nsk: &NullifierSecretKey, - membership_proof: &MembershipProof, - external_seed: Option<&(lee_core::program::PdaSeed, lee_core::program::ProgramId)>, - pos: usize, - pda_seed_by_position: &std::collections::HashMap< - usize, - (lee_core::program::ProgramId, lee_core::program::PdaSeed), - >, - ) { - // With an external seed the binding comes from the circuit input and the - // pre_state is intentionally unauthorized; without one the binding comes from - // a Claim or caller pda_seeds, so the pre_state must already be authorized. - assert!( - self.pre_state.is_authorized ^ external_seed.is_some(), - "PrivatePdaUpdate requires authorized pre_state or external seed" - ); - - let new_nullifier = compute_update_nullifier_and_set_digest( - membership_proof, - &self.pre_state.account, - &self.pre_state.account_id, - nsk, - ); - let new_nonce = self - .pre_state - .account - .nonce - .private_account_nonce_increment(nsk); - - let account_id = self.pre_state.account_id; - let (authority_program_id, seed) = pda_seed_by_position - .get(&pos) - .expect("PrivatePdaUpdate position must be in pda_seed_by_position"); - let kind = PrivateAccountKind::Pda { - program_id: *authority_program_id, - seed: *seed, - identifier: self.identifier, - }; - - self.emit_private_output(&account_id, &kind, new_nullifier, new_nonce); - } - - fn emit_private_output( - self, - account_id: &AccountId, - kind: &PrivateAccountKind, - new_nullifier: (Nullifier, CommitmentSetDigest), - new_nonce: Nonce, - ) { - self.output.new_nullifiers.push(new_nullifier); - - let mut post_with_updated_nonce = self.post_state; - post_with_updated_nonce.nonce = new_nonce; - - let commitment_post = Commitment::new(account_id, &post_with_updated_nonce); - let encrypted_account = EncryptionScheme::encrypt( - &post_with_updated_nonce, - kind, - self.ssk, - &commitment_post, - *self.output_index, - ); - - self.output.new_commitments.push(commitment_post); - self.output - .encrypted_private_post_states - .push(EncryptedAccountData { - ciphertext: encrypted_account, - epk: self.epk.clone(), - view_tag: self.view_tag, - }); - *self.output_index = self - .output_index - .checked_add(1) - .unwrap_or_else(|| panic!("Too many private accounts, output index overflow")); - } -} - -fn init_nullifier_and_nonce( - account_id: &AccountId, - commitment_root: &CommitmentSetDigest, -) -> ((Nullifier, CommitmentSetDigest), Nonce) { - let nullifier = ( - Nullifier::for_account_initialization(account_id), - *commitment_root, - ); - let nonce = Nonce::private_account_nonce_init(account_id); - (nullifier, nonce) -} - -fn derive_and_verify_account_id( - npk: &NullifierPublicKey, - identifier: u128, - pre_state_account_id: AccountId, -) -> AccountId { - let account_id = AccountId::for_regular_private_account(npk, identifier); - assert_eq!(account_id, pre_state_account_id, "AccountId mismatch"); - account_id -} - -fn compute_update_nullifier_and_set_digest( - membership_proof: &MembershipProof, - pre_account: &Account, - account_id: &AccountId, - nsk: &NullifierSecretKey, -) -> (Nullifier, CommitmentSetDigest) { - let commitment_pre = Commitment::new(account_id, pre_account); - let set_digest = compute_digest_for_path(&commitment_pre, membership_proof); - let nullifier = Nullifier::for_account_update(&commitment_pre, nsk); - (nullifier, set_digest) -} - pub fn compute_circuit_output( execution_state: ExecutionState, account_identities: &[InputAccountIdentity], @@ -271,105 +41,290 @@ pub fn compute_circuit_output( output.public_post_states.push(post_state); } InputAccountIdentity::PrivateAuthorizedInit { - epk, - view_tag, - ssk, + vpk, + random_seed, nsk, identifier, commitment_root, - } => PrivateOutputHandler { - output: &mut output, - output_index: &mut output_index, - pre_state: &pre_state, - post_state, - epk, - view_tag: *view_tag, - ssk, - identifier: *identifier, + } => { + let npk = NullifierPublicKey::from(nsk); + let account_id = AccountId::for_regular_private_account(&npk, vpk, *identifier); + + assert_eq!(account_id, pre_state.account_id, "AccountId mismatch"); + assert!( + pre_state.is_authorized, + "Pre-state not authorized for authenticated private account" + ); + assert_eq!( + pre_state.account, + Account::default(), + "Found new private account with non default values" + ); + + let new_nullifier = ( + Nullifier::for_account_initialization(&account_id), + *commitment_root, + ); + let new_nonce = Nonce::private_account_nonce_init(&account_id); + + emit_private_output( + &mut output, + &mut output_index, + post_state, + &account_id, + &PrivateAccountKind::Regular(*identifier), + &npk, + vpk, + random_seed, + new_nullifier, + new_nonce, + ); } - .authorized_init(nsk, commitment_root), InputAccountIdentity::PrivateAuthorizedUpdate { - epk, - view_tag, - ssk, + vpk, + random_seed, nsk, membership_proof, identifier, - } => PrivateOutputHandler { - output: &mut output, - output_index: &mut output_index, - pre_state: &pre_state, - post_state, - epk, - view_tag: *view_tag, - ssk, - identifier: *identifier, + } => { + let npk = NullifierPublicKey::from(nsk); + let account_id = AccountId::for_regular_private_account(&npk, vpk, *identifier); + + assert_eq!(account_id, pre_state.account_id, "AccountId mismatch"); + assert!( + pre_state.is_authorized, + "Pre-state not authorized for authenticated private account" + ); + + let new_nullifier = compute_update_nullifier_and_set_digest( + membership_proof, + &pre_state.account, + &account_id, + nsk, + ); + let new_nonce = pre_state.account.nonce.private_account_nonce_increment(nsk); + + emit_private_output( + &mut output, + &mut output_index, + post_state, + &account_id, + &PrivateAccountKind::Regular(*identifier), + &npk, + vpk, + random_seed, + new_nullifier, + new_nonce, + ); } - .authorized_update(nsk, membership_proof), InputAccountIdentity::PrivateUnauthorized { - epk, - view_tag, + vpk, + random_seed, npk, - ssk, identifier, commitment_root, - } => PrivateOutputHandler { - output: &mut output, - output_index: &mut output_index, - pre_state: &pre_state, - post_state, - epk, - view_tag: *view_tag, - ssk, - identifier: *identifier, + } => { + let account_id = AccountId::for_regular_private_account(npk, vpk, *identifier); + + assert_eq!(account_id, pre_state.account_id, "AccountId mismatch"); + assert_eq!( + pre_state.account, + Account::default(), + "Found new private account with non default values", + ); + assert!( + !pre_state.is_authorized, + "Found new private account marked as authorized." + ); + + let new_nullifier = ( + Nullifier::for_account_initialization(&account_id), + *commitment_root, + ); + let new_nonce = Nonce::private_account_nonce_init(&account_id); + + emit_private_output( + &mut output, + &mut output_index, + post_state, + &account_id, + &PrivateAccountKind::Regular(*identifier), + npk, + vpk, + random_seed, + new_nullifier, + new_nonce, + ); } - .unauthorized(npk, commitment_root), InputAccountIdentity::PrivatePdaInit { - epk, - view_tag, - npk: _, - ssk, + vpk, + random_seed, + npk, identifier, commitment_root, seed: _, - } => PrivateOutputHandler { - output: &mut output, - output_index: &mut output_index, - pre_state: &pre_state, - post_state, - epk, - view_tag: *view_tag, - ssk, - identifier: *identifier, + } => { + // The npk-to-account_id binding is established upstream in + // `validate_and_sync_states` via `Claim::Pda(seed)` or a caller `pda_seeds` + // match. Here we only enforce the init pre-conditions. The supplied npk on + // the variant has been recorded into `private_pda_by_position` and used + // for the binding check; we use `pre_state.account_id` directly for nullifier + // and commitment derivation. + assert!( + !pre_state.is_authorized, + "PrivatePdaInit requires unauthorized pre_state" + ); + assert_eq!( + pre_state.account, + Account::default(), + "New private PDA must be default" + ); + + let new_nullifier = ( + Nullifier::for_account_initialization(&pre_state.account_id), + *commitment_root, + ); + let new_nonce = Nonce::private_account_nonce_init(&pre_state.account_id); + + let account_id = pre_state.account_id; + let (authority_program_id, seed) = pda_seed_by_position + .get(&pos) + .expect("PrivatePdaInit position must be in pda_seed_by_position"); + emit_private_output( + &mut output, + &mut output_index, + post_state, + &account_id, + &PrivateAccountKind::Pda { + program_id: *authority_program_id, + seed: *seed, + identifier: *identifier, + }, + npk, + vpk, + random_seed, + new_nullifier, + new_nonce, + ); } - .pda_init(commitment_root, pos, &pda_seed_by_position), InputAccountIdentity::PrivatePdaUpdate { - epk, - view_tag, - ssk, + vpk, + random_seed, nsk, membership_proof, identifier, seed: external_seed, - } => PrivateOutputHandler { - output: &mut output, - output_index: &mut output_index, - pre_state: &pre_state, - post_state, - epk, - view_tag: *view_tag, - ssk, - identifier: *identifier, + } => { + // With an external seed the binding comes from the circuit input and the + // pre_state is intentionally unauthorized; without one the binding comes from + // a Claim or caller pda_seeds, so the pre_state must already be authorized. + // When `external_seed` is `Some`, execution_state already asserted + // `!pre_state.is_authorized`. + assert!( + pre_state.is_authorized ^ external_seed.is_some(), + "PrivatePdaUpdate requires authorized pre_state or external seed" + ); + + let new_nullifier = compute_update_nullifier_and_set_digest( + membership_proof, + &pre_state.account, + &pre_state.account_id, + nsk, + ); + let new_nonce = pre_state.account.nonce.private_account_nonce_increment(nsk); + + let account_id = pre_state.account_id; + let npk = NullifierPublicKey::from(nsk); + let (authority_program_id, seed) = pda_seed_by_position + .get(&pos) + .expect("PrivatePdaUpdate position must be in pda_seed_by_position"); + emit_private_output( + &mut output, + &mut output_index, + post_state, + &account_id, + &PrivateAccountKind::Pda { + program_id: *authority_program_id, + seed: *seed, + identifier: *identifier, + }, + &npk, + vpk, + random_seed, + new_nullifier, + new_nonce, + ); } - .pda_update( - nsk, - membership_proof, - external_seed.as_ref(), - pos, - &pda_seed_by_position, - ), } } output } + +#[expect( + clippy::too_many_arguments, + reason = "Inputs are distinct concerns from the variant arms; bundling would be artificial" +)] +fn emit_private_output( + output: &mut PrivacyPreservingCircuitOutput, + output_index: &mut u32, + post_state: Account, + account_id: &AccountId, + kind: &PrivateAccountKind, + npk: &NullifierPublicKey, + vpk: &ViewingPublicKey, + random_seed: &[u8; 32], + new_nullifier: (Nullifier, CommitmentSetDigest), + new_nonce: Nonce, +) { + output.new_nullifiers.push(new_nullifier); + + let mut post_with_updated_nonce = post_state; + post_with_updated_nonce.nonce = new_nonce; + + let commitment_post = Commitment::new(account_id, &post_with_updated_nonce); + + let esk = EphemeralSecretKey::new(account_id, random_seed, &new_nonce); + let (shared_secret, epk) = SharedSecretKey::encapsulate_deterministic(vpk, &esk); + + // Currently the view tag is properlty generated for all accounts. + // To increase privacy, this will be changed in the later version + // to only be generated explicitly for initialized accounts and + // fed by the prover directly for updated accounts. + // + // See issue 573: + // https://github.com/logos-blockchain/logos-execution-zone/issues/573 + let view_tag = EncryptedAccountData::compute_view_tag(npk, vpk); + + let encrypted_account = EncryptionScheme::encrypt( + &post_with_updated_nonce, + kind, + &shared_secret, + &commitment_post, + *output_index, + ); + + output.new_commitments.push(commitment_post); + output + .encrypted_private_post_states + .push(EncryptedAccountData { + ciphertext: encrypted_account, + epk, + view_tag, + }); + *output_index = output_index + .checked_add(1) + .unwrap_or_else(|| panic!("Too many private accounts, output index overflow")); +} + +fn compute_update_nullifier_and_set_digest( + membership_proof: &MembershipProof, + pre_account: &Account, + account_id: &AccountId, + nsk: &NullifierSecretKey, +) -> (Nullifier, CommitmentSetDigest) { + let commitment_pre = Commitment::new(account_id, pre_account); + let set_digest = compute_digest_for_path(&commitment_pre, membership_proof); + let nullifier = Nullifier::for_account_update(&commitment_pre, nsk); + (nullifier, set_digest) +} diff --git a/lee/state_machine/Cargo.toml b/lee/state_machine/Cargo.toml index d4014465..df43da8d 100644 --- a/lee/state_machine/Cargo.toml +++ b/lee/state_machine/Cargo.toml @@ -28,10 +28,8 @@ build_utils.workspace = true [dev-dependencies] lee_core = { workspace = true, features = ["test_utils"] } -token_core.workspace = true test_methods = { path = "test_methods" } -env_logger.workspace = true hex-literal = "1.0.0" test-case = "3.3.1" diff --git a/lee/state_machine/core/Cargo.toml b/lee/state_machine/core/Cargo.toml index 6e1f0ff0..585637ae 100644 --- a/lee/state_machine/core/Cargo.toml +++ b/lee/state_machine/core/Cargo.toml @@ -16,7 +16,7 @@ thiserror.workspace = true bytemuck.workspace = true bytesize.workspace = true base58.workspace = true -ml-kem = { workspace = true, optional = true, features = ["getrandom"] } +ml-kem = { workspace = true } chacha20 = { version = "0.10" } [dev-dependencies] @@ -24,5 +24,5 @@ serde_json.workspace = true [features] default = [] -host = ["dep:ml-kem"] +host = ["ml-kem/getrandom"] test_utils = ["host"] diff --git a/lee/state_machine/core/src/circuit_io.rs b/lee/state_machine/core/src/circuit_io.rs index 88b214d4..d441ff19 100644 --- a/lee/state_machine/core/src/circuit_io.rs +++ b/lee/state_machine/core/src/circuit_io.rs @@ -2,9 +2,9 @@ use serde::{Deserialize, Serialize}; use crate::{ Commitment, CommitmentSetDigest, Identifier, MembershipProof, Nullifier, NullifierPublicKey, - NullifierSecretKey, SharedSecretKey, + NullifierSecretKey, account::{Account, AccountWithMetadata}, - encryption::{EncryptedAccountData, EphemeralPublicKey, ViewTag}, + encryption::{EncryptedAccountData, ViewingPublicKey}, program::{BlockValidityWindow, PdaSeed, ProgramId, ProgramOutput, TimestampValidityWindow}, }; @@ -14,15 +14,13 @@ pub struct PrivacyPreservingCircuitInput { pub program_outputs: Vec, /// One entry per `pre_state`, in the same order as the program's `pre_states`. /// Length must equal the number of `pre_states` derived from `program_outputs`. - /// The guest's `private_pda_npk_by_position` and `private_pda_bound_positions` + /// The guest's `private_pda_by_position` and `private_pda_bound_positions` /// rely on this position alignment. pub account_identities: Vec, /// Program ID. pub program_id: ProgramId, } -/// Per-account input to the privacy-preserving circuit. Each variant carries exactly the fields -/// the guest needs for that account's code path. #[derive(Serialize, Deserialize, Clone)] pub enum InputAccountIdentity { /// Public account. The guest reads pre/post state from `program_outputs` and emits no @@ -30,12 +28,11 @@ pub enum InputAccountIdentity { Public, /// Init of an authorized standalone private account: no membership proof. The `pre_state` /// must be `Account::default()`. The `account_id` is derived as - /// `AccountId::for_regular_private_account(&NullifierPublicKey::from(nsk), identifier)` and - /// matched against `pre_state.account_id`. + /// `AccountId::for_regular_private_account(&NullifierPublicKey::from(nsk), vpk, identifier)` + /// and matched against `pre_state.account_id`. PrivateAuthorizedInit { - epk: EphemeralPublicKey, - view_tag: ViewTag, - ssk: SharedSecretKey, + vpk: ViewingPublicKey, + random_seed: [u8; 32], nsk: NullifierSecretKey, identifier: Identifier, commitment_root: CommitmentSetDigest, @@ -43,9 +40,8 @@ pub enum InputAccountIdentity { /// Update of an authorized standalone private account: existing on-chain commitment, with /// membership proof. PrivateAuthorizedUpdate { - epk: EphemeralPublicKey, - view_tag: ViewTag, - ssk: SharedSecretKey, + vpk: ViewingPublicKey, + random_seed: [u8; 32], nsk: NullifierSecretKey, membership_proof: MembershipProof, identifier: Identifier, @@ -53,10 +49,9 @@ pub enum InputAccountIdentity { /// Init of a standalone private account the caller does not own (e.g. a recipient who /// doesn't yet exist on chain). No `nsk`, no membership proof. PrivateUnauthorized { - epk: EphemeralPublicKey, - view_tag: ViewTag, + vpk: ViewingPublicKey, + random_seed: [u8; 32], npk: NullifierPublicKey, - ssk: SharedSecretKey, identifier: Identifier, commitment_root: CommitmentSetDigest, }, @@ -65,15 +60,14 @@ pub enum InputAccountIdentity { /// PDA within the `(program_id, seed, npk)` family: `AccountId::for_private_pda` uses it /// as the 4th input. PrivatePdaInit { - epk: EphemeralPublicKey, - view_tag: ViewTag, + vpk: ViewingPublicKey, + random_seed: [u8; 32], npk: NullifierPublicKey, - ssk: SharedSecretKey, identifier: Identifier, commitment_root: CommitmentSetDigest, /// When `Some((seed, authority_program_id))`, the circuit binds this position via the /// external derivation check - /// `AccountId::for_private_pda(authority_program_id, seed, npk, identifier) == + /// `AccountId::for_private_pda(authority_program_id, seed, npk, vpk, identifier) == /// pre_state.account_id` rather than requiring a `Claim::Pda` or caller /// `pda_seeds` to establish the binding. The `pre_state` must have `is_authorized /// == false`. @@ -83,15 +77,14 @@ pub enum InputAccountIdentity { /// from `nsk`. Authorization may be established upstream by a caller `pda_seeds` match or a /// previously-seen authorization in a chained call. PrivatePdaUpdate { - epk: EphemeralPublicKey, - view_tag: ViewTag, - ssk: SharedSecretKey, + vpk: ViewingPublicKey, + random_seed: [u8; 32], nsk: NullifierSecretKey, membership_proof: MembershipProof, identifier: Identifier, /// When `Some((seed, authority_program_id))`, the circuit binds this position via the /// external derivation check - /// `AccountId::for_private_pda(authority_program_id, seed, npk, identifier) == + /// `AccountId::for_private_pda(authority_program_id, seed, npk, vpk, identifier) == /// pre_state.account_id` rather than requiring a caller `pda_seeds` to establish /// the binding. The `pre_state` must have `is_authorized == false`. seed: Option<(PdaSeed, ProgramId)>, @@ -112,17 +105,23 @@ impl InputAccountIdentity { ) } - /// For private PDA variants, return the `(npk, identifier)` pair. `Init` carries both - /// directly; `Update` derives `npk` from `nsk`. For non-PDA variants returns `None`. #[must_use] - pub fn npk_if_private_pda(&self) -> Option<(NullifierPublicKey, Identifier)> { + pub fn npk_vpk_if_private_pda( + &self, + ) -> Option<(NullifierPublicKey, ViewingPublicKey, Identifier)> { match self { Self::PrivatePdaInit { - npk, identifier, .. - } => Some((*npk, *identifier)), + npk, + vpk, + identifier, + .. + } => Some((*npk, vpk.clone(), *identifier)), Self::PrivatePdaUpdate { - nsk, identifier, .. - } => Some((NullifierPublicKey::from(nsk), *identifier)), + nsk, + vpk, + identifier, + .. + } => Some((NullifierPublicKey::from(nsk), vpk.clone(), *identifier)), Self::Public | Self::PrivateAuthorizedInit { .. } | Self::PrivateAuthorizedUpdate { .. } @@ -161,7 +160,7 @@ mod tests { use crate::{ Commitment, Nullifier, account::{Account, AccountId, AccountWithMetadata, Nonce}, - encryption::Ciphertext, + encryption::{Ciphertext, EphemeralPublicKey}, }; #[test] diff --git a/lee/state_machine/core/src/encryption/mod.rs b/lee/state_machine/core/src/encryption/mod.rs index 47e0995d..f503279d 100644 --- a/lee/state_machine/core/src/encryption/mod.rs +++ b/lee/state_machine/core/src/encryption/mod.rs @@ -5,11 +5,9 @@ use chacha20::{ }; use risc0_zkvm::sha::{Impl, Sha256 as _}; use serde::{Deserialize, Serialize}; -#[cfg(feature = "host")] pub use shared_key_derivation::{MlKem768EncapsulationKey, ViewingPublicKey}; use crate::{Commitment, account::Account, program::PrivateAccountKind}; -#[cfg(feature = "host")] pub mod shared_key_derivation; /// Length in bytes of an ML-KEM-768 ciphertext (the `EphemeralPublicKey` payload). @@ -17,6 +15,31 @@ pub const ML_KEM_768_CIPHERTEXT_LEN: usize = 1088; pub type Scalar = [u8; 32]; +#[derive(Serialize, Deserialize, Clone, Copy)] +pub struct EphemeralSecretKey(pub [u8; 32]); + +impl EphemeralSecretKey { + /// Derives an ephemeral secret key from OS randomness and account-specific values. + /// + /// For updates, `nonce` carries `nsk`-derived entropy, making `esk` strong even + /// with a compromised RNG. For inits, `nonce` is deterministic, so `random_seed` + /// is the sole entropy source. + #[must_use] + pub fn new( + account_id: &crate::account::AccountId, + random_seed: &[u8; 32], + nonce: &crate::account::Nonce, + ) -> Self { + const PREFIX: &[u8; 14] = b"/LEE/v0.3/esk/"; + let mut input = [0_u8; 14 + 32 + 32 + 16]; + input[0..14].copy_from_slice(PREFIX); + input[14..46].copy_from_slice(account_id.value()); + input[46..78].copy_from_slice(random_seed); + input[78..94].copy_from_slice(&nonce.0.to_le_bytes()); + Self(Impl::hash_bytes(&input).as_bytes().try_into().unwrap()) + } +} + #[derive(Serialize, Deserialize, Clone, Copy)] pub struct SharedSecretKey(pub [u8; 32]); @@ -55,6 +78,18 @@ pub struct EncryptedAccountData { pub view_tag: ViewTag, } +impl EncryptedAccountData { + #[must_use] + pub fn compute_view_tag(npk: &crate::NullifierPublicKey, vpk: &ViewingPublicKey) -> ViewTag { + const PREFIX: &[u8; 18] = b"/LEE/v0.3/ViewTag/"; + let mut bytes = [0_u8; 18 + 32 + ViewingPublicKey::LEN]; + bytes[0..18].copy_from_slice(PREFIX); + bytes[18..50].copy_from_slice(&npk.to_byte_array()); + bytes[50..].copy_from_slice(vpk.to_bytes()); + Impl::hash_bytes(&bytes).as_bytes()[0] + } +} + #[cfg(feature = "host")] impl EncryptedAccountData { #[must_use] @@ -71,16 +106,6 @@ impl EncryptedAccountData { view_tag, } } - - /// Computes the tag as the first byte of SHA256("/LEE/v0.3/ViewTag/" || npk || vpk). - #[must_use] - pub fn compute_view_tag(npk: &crate::NullifierPublicKey, vpk: &ViewingPublicKey) -> ViewTag { - let mut bytes = Vec::new(); - bytes.extend_from_slice(b"/LEE/v0.3/ViewTag/"); - bytes.extend_from_slice(&npk.to_byte_array()); - bytes.extend_from_slice(vpk.to_bytes()); - Impl::hash_bytes(&bytes).as_bytes()[0] - } } impl EncryptionScheme { @@ -116,12 +141,12 @@ impl EncryptionScheme { commitment: &Commitment, output_index: u32, ) -> [u8; 32] { - let mut bytes = Vec::new(); - - bytes.extend_from_slice(b"LEE/v0.2/KDF-SHA256/"); - bytes.extend_from_slice(&shared_secret.0); - bytes.extend_from_slice(&commitment.to_byte_array()); - bytes.extend_from_slice(&output_index.to_le_bytes()); + const PREFIX: &[u8; 20] = b"LEE/v0.2/KDF-SHA256/"; + let mut bytes = [0_u8; 20 + 32 + 32 + 4]; + bytes[0..20].copy_from_slice(PREFIX); + bytes[20..52].copy_from_slice(&shared_secret.0); + bytes[52..84].copy_from_slice(&commitment.to_byte_array()); + bytes[84..88].copy_from_slice(&output_index.to_le_bytes()); Impl::hash_bytes(&bytes).as_bytes().try_into().unwrap() } @@ -238,4 +263,41 @@ mod tests { "wrong shared secret must not produce the correct plaintext" ); } + + #[test] + fn esk_is_deterministic() { + let account_id = AccountId::new([1_u8; 32]); + let random_seed = [2_u8; 32]; + let nonce = crate::account::Nonce(42); + let esk1 = EphemeralSecretKey::new(&account_id, &random_seed, &nonce); + let esk2 = EphemeralSecretKey::new(&account_id, &random_seed, &nonce); + assert_eq!(esk1.0, esk2.0); + } + + #[test] + fn esk_differs_for_different_account_id() { + let random_seed = [2_u8; 32]; + let nonce = crate::account::Nonce(42); + let esk_a = EphemeralSecretKey::new(&AccountId::new([0_u8; 32]), &random_seed, &nonce); + let esk_b = EphemeralSecretKey::new(&AccountId::new([1_u8; 32]), &random_seed, &nonce); + assert_ne!(esk_a.0, esk_b.0); + } + + #[test] + fn esk_differs_for_different_random_seed() { + let account_id = AccountId::new([1_u8; 32]); + let nonce = crate::account::Nonce(42); + let esk_a = EphemeralSecretKey::new(&account_id, &[0_u8; 32], &nonce); + let esk_b = EphemeralSecretKey::new(&account_id, &[1_u8; 32], &nonce); + assert_ne!(esk_a.0, esk_b.0); + } + + #[test] + fn esk_differs_for_different_nonce() { + let account_id = AccountId::new([1_u8; 32]); + let random_seed = [2_u8; 32]; + let esk_a = EphemeralSecretKey::new(&account_id, &random_seed, &crate::account::Nonce(0)); + let esk_b = EphemeralSecretKey::new(&account_id, &random_seed, &crate::account::Nonce(1)); + assert_ne!(esk_a.0, esk_b.0); + } } diff --git a/lee/state_machine/core/src/encryption/shared_key_derivation.rs b/lee/state_machine/core/src/encryption/shared_key_derivation.rs index 4b46165a..3255080b 100644 --- a/lee/state_machine/core/src/encryption/shared_key_derivation.rs +++ b/lee/state_machine/core/src/encryption/shared_key_derivation.rs @@ -1,5 +1,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; -use ml_kem::{Decapsulate as _, Encapsulate as _, KeyExport as _, Seed}; +#[cfg(feature = "host")] +use ml_kem::Encapsulate as _; +use ml_kem::{Decapsulate as _, KeyExport as _, Seed}; use serde::{Deserialize, Serialize}; use crate::{EphemeralPublicKey, SharedSecretKey}; @@ -26,6 +28,7 @@ impl MlKem768EncapsulationKey { pub const LEN: usize = 1184; /// Construct from raw bytes, returning an error if the length is not [`Self::LEN`]. + #[cfg(feature = "host")] pub fn from_bytes(bytes: Vec) -> Result { if bytes.len() != Self::LEN { return Err(crate::error::LeeCoreError::DeserializationError(format!( @@ -59,6 +62,7 @@ impl SharedSecretKey { /// Returns `(shared_secret, ciphertext)`. The ciphertext must be included in the transaction /// as the `EphemeralPublicKey`; the receiver recovers the same shared secret via /// [`Self::decapsulate`]. + #[cfg(feature = "host")] #[must_use] pub fn encapsulate(ek: &MlKem768EncapsulationKey) -> (Self, EphemeralPublicKey) { let ek_bytes: ml_kem::kem::Key = @@ -76,30 +80,18 @@ impl SharedSecretKey { (Self(ss_bytes), EphemeralPublicKey(ct.to_vec())) } - /// Deterministically encapsulate a shared secret toward `ek` for use in tests. + /// Deterministically encapsulate a shared secret toward `ek` using a + /// pre-derived `esk` as the ML-KEM encapsulation randomness. /// - /// The shared secret has no secret entropy โ€” it is fully determined by `ek`, - /// `message_hash`, and `output_index`, all of which are public. This makes it - /// unsuitable for real encryption but useful for producing stable, reproducible - /// shared secrets in unit tests. Use a distinct `output_index` per output to - /// avoid EPK collisions across multiple outputs in the same test. - /// - /// For production use [`Self::encapsulate`], which draws randomness from the OS. - #[cfg(any(test, feature = "test_utils"))] + /// The `esk` must be derived via `derive_esk(account_id, random_seed, nonce)` + /// which binds it to the account and incorporates OS entropy. #[must_use] pub fn encapsulate_deterministic( ek: &MlKem768EncapsulationKey, - message_hash: &[u8; 32], - output_index: u32, + esk: &crate::encryption::EphemeralSecretKey, ) -> (Self, EphemeralPublicKey) { - use risc0_zkvm::sha::{Impl, Sha256 as _}; - - let mut input = Vec::with_capacity(36); - input.extend_from_slice(message_hash); - input.extend_from_slice(&output_index.to_le_bytes()); - let hash = Impl::hash_bytes(&input); - let m: ml_kem::B32 = - ml_kem::array::Array::try_from(hash.as_bytes()).expect("SHA-256 output is 32 bytes"); + let m: ml_kem::B32 = ml_kem::array::Array::try_from(esk.0.as_slice()) + .expect("EphemeralSecretKey is 32 bytes"); let ek_bytes: ml_kem::kem::Key = ek.0.as_slice() diff --git a/lee/state_machine/core/src/lib.rs b/lee/state_machine/core/src/lib.rs index 900de667..4bcb23a4 100644 --- a/lee/state_machine/core/src/lib.rs +++ b/lee/state_machine/core/src/lib.rs @@ -11,8 +11,8 @@ pub use commitment::{ compute_digest_for_path, }; pub use encryption::{ - EncryptedAccountData, EncryptionScheme, EphemeralPublicKey, ML_KEM_768_CIPHERTEXT_LEN, - SharedSecretKey, ViewTag, + EncryptedAccountData, EncryptionScheme, EphemeralPublicKey, EphemeralSecretKey, + ML_KEM_768_CIPHERTEXT_LEN, SharedSecretKey, ViewTag, }; pub use nullifier::{Identifier, Nullifier, NullifierPublicKey, NullifierSecretKey}; pub use program::PrivateAccountKind; diff --git a/lee/state_machine/core/src/nullifier.rs b/lee/state_machine/core/src/nullifier.rs index 0490ac00..0374a31a 100644 --- a/lee/state_machine/core/src/nullifier.rs +++ b/lee/state_machine/core/src/nullifier.rs @@ -2,7 +2,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use risc0_zkvm::sha::{Impl, Sha256 as _}; use serde::{Deserialize, Serialize}; -use crate::{Commitment, account::AccountId}; +use crate::{Commitment, account::AccountId, encryption::ViewingPublicKey}; const PRIVATE_ACCOUNT_ID_PREFIX: &[u8; 32] = b"/LEE/v0.3/AccountId/Private/\x00\x00\x00\x00"; @@ -16,12 +16,16 @@ impl AccountId { /// Derives an [`AccountId`] for a regular (non-PDA) private account from the nullifier public /// key and identifier. #[must_use] - pub fn for_regular_private_account(npk: &NullifierPublicKey, identifier: Identifier) -> Self { - // 32 bytes prefix || 32 bytes npk || 16 bytes identifier - let mut bytes = [0; 80]; + pub fn for_regular_private_account( + npk: &NullifierPublicKey, + vpk: &ViewingPublicKey, + identifier: Identifier, + ) -> Self { + let mut bytes = [0_u8; 32 + 32 + ViewingPublicKey::LEN + 16]; bytes[0..32].copy_from_slice(PRIVATE_ACCOUNT_ID_PREFIX); bytes[32..64].copy_from_slice(&npk.0); - bytes[64..80].copy_from_slice(&identifier.to_le_bytes()); + bytes[64..64 + ViewingPublicKey::LEN].copy_from_slice(vpk.to_bytes()); + bytes[64 + ViewingPublicKey::LEN..].copy_from_slice(&identifier.to_le_bytes()); Self::new( Impl::hash_bytes(&bytes) @@ -32,9 +36,9 @@ impl AccountId { } } -impl From<(&NullifierPublicKey, Identifier)> for AccountId { - fn from((npk, identifier): (&NullifierPublicKey, Identifier)) -> Self { - Self::for_regular_private_account(npk, identifier) +impl From<(&NullifierPublicKey, &ViewingPublicKey, Identifier)> for AccountId { + fn from((npk, vpk, identifier): (&NullifierPublicKey, &ViewingPublicKey, Identifier)) -> Self { + Self::for_regular_private_account(npk, vpk, identifier) } } @@ -158,12 +162,13 @@ mod tests { 196, 134, 22, 224, 211, 237, 120, 136, 225, 188, 220, 249, 28, ]; let npk = NullifierPublicKey::from(&nsk); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); let expected_account_id = AccountId::new([ - 165, 52, 40, 32, 231, 171, 113, 10, 65, 241, 156, 72, 154, 207, 122, 192, 15, 46, 50, - 253, 105, 164, 89, 84, 40, 191, 182, 119, 64, 255, 67, 142, + 242, 239, 57, 244, 89, 109, 65, 201, 223, 100, 43, 87, 205, 83, 148, 161, 176, 22, 208, + 220, 68, 135, 10, 171, 182, 80, 54, 74, 228, 244, 236, 7, ]); - let account_id = AccountId::for_regular_private_account(&npk, 0); + let account_id = AccountId::for_regular_private_account(&npk, &vpk, 0); assert_eq!(account_id, expected_account_id); } @@ -175,12 +180,13 @@ mod tests { 196, 134, 22, 224, 211, 237, 120, 136, 225, 188, 220, 249, 28, ]; let npk = NullifierPublicKey::from(&nsk); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); let expected_account_id = AccountId::new([ - 203, 201, 109, 245, 40, 54, 195, 12, 55, 33, 0, 86, 245, 65, 70, 156, 24, 249, 26, 95, - 56, 247, 99, 121, 165, 182, 234, 255, 19, 127, 191, 72, + 149, 125, 157, 109, 119, 81, 9, 163, 231, 181, 214, 43, 57, 113, 221, 72, 180, 149, + 189, 170, 32, 181, 255, 231, 19, 92, 235, 59, 153, 185, 172, 206, ]); - let account_id = AccountId::for_regular_private_account(&npk, 1); + let account_id = AccountId::for_regular_private_account(&npk, &vpk, 1); assert_eq!(account_id, expected_account_id); } @@ -193,12 +199,13 @@ mod tests { 196, 134, 22, 224, 211, 237, 120, 136, 225, 188, 220, 249, 28, ]; let npk = NullifierPublicKey::from(&nsk); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); let expected_account_id = AccountId::new([ - 178, 16, 226, 206, 217, 38, 38, 45, 155, 240, 226, 253, 168, 87, 146, 70, 72, 32, 174, - 19, 245, 25, 214, 162, 209, 135, 252, 82, 27, 2, 174, 196, + 30, 232, 222, 201, 233, 125, 124, 194, 58, 39, 121, 96, 185, 84, 168, 109, 80, 111, + 159, 112, 84, 100, 133, 244, 16, 34, 221, 35, 128, 131, 98, 159, ]); - let account_id = AccountId::for_regular_private_account(&npk, identifier); + let account_id = AccountId::for_regular_private_account(&npk, &vpk, identifier); assert_eq!(account_id, expected_account_id); } diff --git a/lee/state_machine/core/src/program/mod.rs b/lee/state_machine/core/src/program/mod.rs index a4574b82..0d96642e 100644 --- a/lee/state_machine/core/src/program/mod.rs +++ b/lee/state_machine/core/src/program/mod.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use crate::{ BlockId, Identifier, NullifierPublicKey, Timestamp, account::{Account, AccountId, AccountWithMetadata}, + encryption::ViewingPublicKey, }; pub const DEFAULT_PROGRAM_ID: ProgramId = [0; 8]; @@ -154,19 +155,21 @@ impl AccountId { program_id: &ProgramId, seed: &PdaSeed, npk: &NullifierPublicKey, + vpk: &ViewingPublicKey, identifier: Identifier, ) -> Self { use risc0_zkvm::sha::{Impl, Sha256 as _}; const PRIVATE_PDA_PREFIX: &[u8; 32] = b"/LEE/v0.3/AccountId/PrivatePDA/\x00"; - let mut bytes = [0_u8; 144]; + let mut bytes = [0_u8; 32 + 32 + 32 + 32 + ViewingPublicKey::LEN + 16]; bytes[0..32].copy_from_slice(PRIVATE_PDA_PREFIX); let program_id_bytes: &[u8] = bytemuck::try_cast_slice(program_id).expect("ProgramId should be castable to &[u8]"); bytes[32..64].copy_from_slice(program_id_bytes); bytes[64..96].copy_from_slice(&seed.0); bytes[96..128].copy_from_slice(&npk.to_byte_array()); - bytes[128..144].copy_from_slice(&identifier.to_le_bytes()); + bytes[128..128 + ViewingPublicKey::LEN].copy_from_slice(vpk.to_bytes()); + bytes[128 + ViewingPublicKey::LEN..].copy_from_slice(&identifier.to_le_bytes()); Self::new( Impl::hash_bytes(&bytes) .as_bytes() @@ -177,16 +180,20 @@ impl AccountId { /// Derives the [`AccountId`] for a private account from the nullifier public key and kind. #[must_use] - pub fn for_private_account(npk: &NullifierPublicKey, kind: &PrivateAccountKind) -> Self { + pub fn for_private_account( + npk: &NullifierPublicKey, + vpk: &ViewingPublicKey, + kind: &PrivateAccountKind, + ) -> Self { match kind { PrivateAccountKind::Regular(identifier) => { - Self::for_regular_private_account(npk, *identifier) + Self::for_regular_private_account(npk, vpk, *identifier) } PrivateAccountKind::Pda { program_id, seed, identifier, - } => Self::for_private_pda(program_id, seed, npk, *identifier), + } => Self::for_private_pda(program_id, seed, npk, vpk, *identifier), } } } diff --git a/lee/state_machine/core/src/program/tests.rs b/lee/state_machine/core/src/program/tests.rs index df9f5126..19a259a8 100644 --- a/lee/state_machine/core/src/program/tests.rs +++ b/lee/state_machine/core/src/program/tests.rs @@ -184,13 +184,14 @@ fn for_private_pda_matches_pinned_value() { let program_id: ProgramId = [1; 8]; let seed = PdaSeed::new([2; 32]); let npk = NullifierPublicKey([3; 32]); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); let identifier: Identifier = u128::MAX; let expected = AccountId::new([ - 59, 239, 182, 97, 14, 220, 96, 115, 238, 133, 143, 33, 234, 82, 237, 255, 148, 110, 54, - 124, 98, 159, 245, 101, 146, 182, 150, 54, 37, 62, 25, 17, + 5, 87, 128, 244, 206, 244, 65, 130, 178, 88, 225, 183, 0, 159, 201, 201, 212, 206, 6, 156, + 13, 55, 32, 139, 91, 222, 209, 83, 172, 148, 123, 179, ]); assert_eq!( - AccountId::for_private_pda(&program_id, &seed, &npk, identifier), + AccountId::for_private_pda(&program_id, &seed, &npk, &vpk, identifier), expected ); } @@ -202,9 +203,10 @@ fn for_private_pda_differs_for_different_npk() { let seed = PdaSeed::new([2; 32]); let npk_a = NullifierPublicKey([3; 32]); let npk_b = NullifierPublicKey([4; 32]); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); assert_ne!( - AccountId::for_private_pda(&program_id, &seed, &npk_a, u128::MAX), - AccountId::for_private_pda(&program_id, &seed, &npk_b, u128::MAX), + AccountId::for_private_pda(&program_id, &seed, &npk_a, &vpk, u128::MAX), + AccountId::for_private_pda(&program_id, &seed, &npk_b, &vpk, u128::MAX), ); } @@ -215,9 +217,10 @@ fn for_private_pda_differs_for_different_seed() { let seed_a = PdaSeed::new([2; 32]); let seed_b = PdaSeed::new([5; 32]); let npk = NullifierPublicKey([3; 32]); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); assert_ne!( - AccountId::for_private_pda(&program_id, &seed_a, &npk, u128::MAX), - AccountId::for_private_pda(&program_id, &seed_b, &npk, u128::MAX), + AccountId::for_private_pda(&program_id, &seed_a, &npk, &vpk, u128::MAX), + AccountId::for_private_pda(&program_id, &seed_b, &npk, &vpk, u128::MAX), ); } @@ -228,9 +231,10 @@ fn for_private_pda_differs_for_different_program_id() { let program_id_b: ProgramId = [9; 8]; let seed = PdaSeed::new([2; 32]); let npk = NullifierPublicKey([3; 32]); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); assert_ne!( - AccountId::for_private_pda(&program_id_a, &seed, &npk, u128::MAX), - AccountId::for_private_pda(&program_id_b, &seed, &npk, u128::MAX), + AccountId::for_private_pda(&program_id_a, &seed, &npk, &vpk, u128::MAX), + AccountId::for_private_pda(&program_id_b, &seed, &npk, &vpk, u128::MAX), ); } @@ -241,13 +245,14 @@ fn for_private_pda_differs_for_different_identifier() { let program_id: ProgramId = [1; 8]; let seed = PdaSeed::new([2; 32]); let npk = NullifierPublicKey([3; 32]); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); assert_ne!( - AccountId::for_private_pda(&program_id, &seed, &npk, 0), - AccountId::for_private_pda(&program_id, &seed, &npk, 1), + AccountId::for_private_pda(&program_id, &seed, &npk, &vpk, 0), + AccountId::for_private_pda(&program_id, &seed, &npk, &vpk, 1), ); assert_ne!( - AccountId::for_private_pda(&program_id, &seed, &npk, 0), - AccountId::for_private_pda(&program_id, &seed, &npk, u128::MAX), + AccountId::for_private_pda(&program_id, &seed, &npk, &vpk, 0), + AccountId::for_private_pda(&program_id, &seed, &npk, &vpk, u128::MAX), ); } @@ -258,7 +263,8 @@ fn for_private_pda_differs_from_public_pda() { let program_id: ProgramId = [1; 8]; let seed = PdaSeed::new([2; 32]); let npk = NullifierPublicKey([3; 32]); - let private_id = AccountId::for_private_pda(&program_id, &seed, &npk, u128::MAX); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); + let private_id = AccountId::for_private_pda(&program_id, &seed, &npk, &vpk, u128::MAX); let public_id = AccountId::for_public_pda(&program_id, &seed); assert_ne!(private_id, public_id); } @@ -295,22 +301,24 @@ fn for_private_account_dispatches_correctly() { let program_id: ProgramId = [1; 8]; let seed = PdaSeed::new([2; 32]); let npk = NullifierPublicKey([3; 32]); + let vpk = ViewingPublicKey::from_seed(&[1_u8; 32], &[2_u8; 32]); let identifier: Identifier = 77; assert_eq!( - AccountId::for_private_account(&npk, &PrivateAccountKind::Regular(identifier)), - AccountId::for_regular_private_account(&npk, identifier), + AccountId::for_private_account(&npk, &vpk, &PrivateAccountKind::Regular(identifier)), + AccountId::for_regular_private_account(&npk, &vpk, identifier), ); assert_eq!( AccountId::for_private_account( &npk, + &vpk, &PrivateAccountKind::Pda { program_id, seed, identifier } ), - AccountId::for_private_pda(&program_id, &seed, &npk, identifier), + AccountId::for_private_pda(&program_id, &seed, &npk, &vpk, identifier), ); } diff --git a/lee/state_machine/src/privacy_preserving_transaction/circuit/tests.rs b/lee/state_machine/src/privacy_preserving_transaction/circuit/tests.rs index 970bfbd4..005a9230 100644 --- a/lee/state_machine/src/privacy_preserving_transaction/circuit/tests.rs +++ b/lee/state_machine/src/privacy_preserving_transaction/circuit/tests.rs @@ -1,8 +1,8 @@ #![expect(clippy::shadow_unrelated, reason = "We don't care about it in tests")] use lee_core::{ - Commitment, DUMMY_COMMITMENT_HASH, EncryptedAccountData, EncryptionScheme, EphemeralPublicKey, - Nullifier, PrivacyPreservingCircuitOutput, SharedSecretKey, + Commitment, DUMMY_COMMITMENT_HASH, EncryptionScheme, EphemeralSecretKey, Nullifier, + PrivacyPreservingCircuitOutput, SharedSecretKey, account::{Account, AccountId, AccountWithMetadata, Nonce, data::Data}, program::{PdaSeed, PrivateAccountKind}, }; @@ -58,7 +58,8 @@ fn prove_privacy_preserving_execution_circuit_public_and_private_pre_accounts() AccountId::new([0; 32]), ); - let recipient_account_id = AccountId::for_regular_private_account(&recipient_keys.npk(), 0); + let recipient_account_id = + AccountId::for_regular_private_account(&recipient_keys.npk(), &recipient_keys.vpk(), 0); let recipient = AccountWithMetadata::new(Account::default(), false, recipient_account_id); let balance_to_move: u128 = 37; @@ -79,8 +80,9 @@ fn prove_privacy_preserving_execution_circuit_public_and_private_pre_accounts() let expected_sender_pre = sender.clone(); - let shared_secret = - SharedSecretKey::encapsulate_deterministic(&recipient_keys.vpk(), &[0_u8; 32], 0).0; + let init_nonce = Nonce::private_account_nonce_init(&recipient_account_id); + let esk = EphemeralSecretKey::new(&recipient_account_id, &[0; 32], &init_nonce); + let shared_secret = SharedSecretKey::encapsulate_deterministic(&recipient_keys.vpk(), &esk).0; let (output, proof) = execute_and_prove( vec![sender, recipient], @@ -88,13 +90,9 @@ fn prove_privacy_preserving_execution_circuit_public_and_private_pre_accounts() vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: shared_secret, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }, @@ -138,12 +136,14 @@ fn prove_privacy_preserving_execution_circuit_fully_private() { data: Data::default(), }, true, - AccountId::for_regular_private_account(&sender_keys.npk(), 0), + AccountId::for_regular_private_account(&sender_keys.npk(), &sender_keys.vpk(), 0), ); - let sender_account_id = AccountId::for_regular_private_account(&sender_keys.npk(), 0); + let sender_account_id = + AccountId::for_regular_private_account(&sender_keys.npk(), &sender_keys.vpk(), 0); let commitment_sender = Commitment::new(&sender_account_id, &sender_pre.account); - let recipient_account_id = AccountId::for_regular_private_account(&recipient_keys.npk(), 0); + let recipient_account_id = + AccountId::for_regular_private_account(&recipient_keys.npk(), &recipient_keys.vpk(), 0); let recipient = AccountWithMetadata::new(Account::default(), false, recipient_account_id); let balance_to_move: u128 = 37; @@ -179,23 +179,25 @@ fn prove_privacy_preserving_execution_circuit_fully_private() { Commitment::new(&recipient_account_id, &expected_private_account_2), ]; - let shared_secret_1 = - SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0).0; + let esk_1 = EphemeralSecretKey::new( + &sender_account_id, + &[0; 32], + &sender_nonce.private_account_nonce_increment(&sender_keys.nsk), + ); + let shared_secret_1 = SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &esk_1).0; + let init_nonce_2 = Nonce::private_account_nonce_init(&recipient_account_id); + let esk_2 = EphemeralSecretKey::new(&recipient_account_id, &[0; 32], &init_nonce_2); let shared_secret_2 = - SharedSecretKey::encapsulate_deterministic(&recipient_keys.vpk(), &[0_u8; 32], 1).0; + SharedSecretKey::encapsulate_deterministic(&recipient_keys.vpk(), &esk_2).0; let (output, proof) = execute_and_prove( vec![sender_pre, recipient], Program::serialize_instruction(balance_to_move).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: shared_secret_1, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: commitment_set .get_proof_for(&commitment_sender) @@ -203,13 +205,9 @@ fn prove_privacy_preserving_execution_circuit_fully_private() { identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: shared_secret_2, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }, @@ -250,7 +248,7 @@ fn circuit_fails_when_chained_validity_windows_have_empty_intersection() { let pre = AccountWithMetadata::new( Account::default(), false, - AccountId::for_regular_private_account(&account_keys.npk(), 0), + AccountId::for_regular_private_account(&account_keys.npk(), &account_keys.vpk(), 0), ); let validity_window_chain_caller = crate::test_methods::validity_window_chain_caller(); @@ -265,9 +263,6 @@ fn circuit_fails_when_chained_validity_windows_have_empty_intersection() { )) .unwrap(); - let shared_secret = - SharedSecretKey::encapsulate_deterministic(&account_keys.vpk(), &[0_u8; 32], 0).0; - let program_with_deps = ProgramWithDependencies::new( validity_window_chain_caller, [(validity_window.id(), validity_window)].into(), @@ -277,13 +272,9 @@ fn circuit_fails_when_chained_validity_windows_have_empty_intersection() { vec![pre], instruction, vec![InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &account_keys.npk(), - &account_keys.vpk(), - ), + vpk: account_keys.vpk(), + random_seed: [0; 32], npk: account_keys.npk(), - ssk: shared_secret, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }], @@ -302,19 +293,21 @@ fn private_pda_claim_with_custom_identifier_encrypts_correct_kind() { let npk = keys.npk(); let seed = PdaSeed::new([42; 32]); let identifier: u128 = 99; - let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; + let account_id = + AccountId::for_private_pda(&program.id(), &seed, &npk, &keys.vpk(), identifier); + let init_nonce = Nonce::private_account_nonce_init(&account_id); + let esk = EphemeralSecretKey::new(&account_id, &[0; 32], &init_nonce); + let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &esk).0; - let account_id = AccountId::for_private_pda(&program.id(), &seed, &npk, identifier); let pre_state = AccountWithMetadata::new(Account::default(), false, account_id); let (output, _proof) = execute_and_prove( vec![pre_state], Program::serialize_instruction(seed).unwrap(), vec![InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret, identifier, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -343,11 +336,8 @@ fn private_pda_init() { let keys = test_private_account_keys_1(); let npk = keys.npk(); let seed = PdaSeed::new([42; 32]); - let shared_secret_pda = - SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - // PDA (new, private PDA) - let pda_id = AccountId::for_private_pda(&program.id(), &seed, &npk, 0); + let pda_id = AccountId::for_private_pda(&program.id(), &seed, &npk, &keys.vpk(), 0); let pda_pre = AccountWithMetadata::new(Account::default(), false, pda_id); let auth_id = simple_transfer.id(); @@ -361,10 +351,9 @@ fn private_pda_init() { vec![pda_pre], instruction, vec![InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret_pda, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -386,11 +375,8 @@ fn private_pda_withdraw() { let keys = test_private_account_keys_1(); let npk = keys.npk(); let seed = PdaSeed::new([42; 32]); - let shared_secret_pda = - SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - // PDA (new, private PDA) - let pda_id = AccountId::for_private_pda(&program.id(), &seed, &npk, 0); + let pda_id = AccountId::for_private_pda(&program.id(), &seed, &npk, &keys.vpk(), 0); let pda_pre = AccountWithMetadata::new(Account::default(), false, pda_id); // Recipient (public) @@ -417,10 +403,9 @@ fn private_pda_withdraw() { instruction, vec![ InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret_pda, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -445,8 +430,6 @@ fn shared_account_receives_via_simple_transfer() { let shared_keys = test_private_account_keys_1(); let shared_npk = shared_keys.npk(); let shared_identifier: u128 = 42; - let shared_secret = - SharedSecretKey::encapsulate_deterministic(&shared_keys.vpk(), &[0_u8; 32], 0).0; // Sender: public account with balance, owned by auth-transfer let sender_id = AccountId::new([99; 32]); @@ -461,7 +444,7 @@ fn shared_account_receives_via_simple_transfer() { ); // Recipient: shared private account (new, unauthorized) - let shared_account_id = AccountId::from((&shared_npk, shared_identifier)); + let shared_account_id = AccountId::from((&shared_npk, &shared_keys.vpk(), shared_identifier)); let recipient = AccountWithMetadata::new(Account::default(), false, shared_account_id); let balance_to_move: u128 = 100; @@ -473,10 +456,9 @@ fn shared_account_receives_via_simple_transfer() { vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&shared_npk, &shared_keys.vpk()), + vpk: shared_keys.vpk(), + random_seed: [0; 32], npk: shared_npk, - ssk: shared_secret, identifier: shared_identifier, commitment_root: DUMMY_COMMITMENT_HASH, }, @@ -496,17 +478,21 @@ fn private_authorized_init_encrypts_regular_kind_with_identifier() { let program = crate::test_methods::claimer(); let keys = test_private_account_keys_1(); let identifier: u128 = 99; - let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - let account_id = AccountId::for_regular_private_account(&keys.npk(), identifier); + let account_id = AccountId::for_regular_private_account(&keys.npk(), &keys.vpk(), identifier); + let esk = EphemeralSecretKey::new( + &account_id, + &[0; 32], + &Nonce::private_account_nonce_init(&account_id), + ); + let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &esk).0; let pre = AccountWithMetadata::new(Account::default(), true, account_id); let (output, _) = execute_and_prove( vec![pre], Program::serialize_instruction(()).unwrap(), vec![InputAccountIdentity::PrivateAuthorizedInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&keys.npk(), &keys.vpk()), - ssk, + vpk: keys.vpk(), + random_seed: [0; 32], nsk: keys.nsk, identifier, commitment_root: DUMMY_COMMITMENT_HASH, @@ -528,18 +514,22 @@ fn private_unauthorized_init_encrypts_regular_kind_with_identifier() { let program = crate::test_methods::claimer(); let keys = test_private_account_keys_1(); let identifier: u128 = 99; - let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - let recipient_id = AccountId::for_regular_private_account(&keys.npk(), identifier); + let recipient_id = AccountId::for_regular_private_account(&keys.npk(), &keys.vpk(), identifier); + let esk = EphemeralSecretKey::new( + &recipient_id, + &[0; 32], + &Nonce::private_account_nonce_init(&recipient_id), + ); + let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &esk).0; let recipient = AccountWithMetadata::new(Account::default(), false, recipient_id); let (output, _) = execute_and_prove( vec![recipient], Program::serialize_instruction(()).unwrap(), vec![InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&keys.npk(), &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk: keys.npk(), - ssk, identifier, commitment_root: DUMMY_COMMITMENT_HASH, }], @@ -560,8 +550,13 @@ fn private_authorized_update_encrypts_regular_kind_with_identifier() { let program = crate::test_methods::noop(); let keys = test_private_account_keys_1(); let identifier: u128 = 99; - let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - let account_id = AccountId::for_regular_private_account(&keys.npk(), identifier); + let account_id = AccountId::for_regular_private_account(&keys.npk(), &keys.vpk(), identifier); + let esk = EphemeralSecretKey::new( + &account_id, + &[0; 32], + &Nonce::default().private_account_nonce_increment(&keys.nsk), + ); + let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &esk).0; let account = Account { program_owner: program.id(), balance: 1, @@ -577,9 +572,8 @@ fn private_authorized_update_encrypts_regular_kind_with_identifier() { vec![sender], Program::serialize_instruction(()).unwrap(), vec![InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&keys.npk(), &keys.vpk()), - ssk, + vpk: keys.vpk(), + random_seed: [0; 32], nsk: keys.nsk, membership_proof: commitment_set.get_proof_for(&commitment).unwrap(), identifier, @@ -604,10 +598,14 @@ fn private_pda_update_encrypts_pda_kind_with_identifier() { let npk = keys.npk(); let seed = PdaSeed::new([42; 32]); let identifier: u128 = 99; - let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - let simple_transfer_id = simple_transfer.id(); - let pda_id = AccountId::for_private_pda(&program.id(), &seed, &npk, identifier); + let pda_id = AccountId::for_private_pda(&program.id(), &seed, &npk, &keys.vpk(), identifier); + let esk = EphemeralSecretKey::new( + &pda_id, + &[0; 32], + &Nonce::default().private_account_nonce_increment(&keys.nsk), + ); + let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &esk).0; let pda_account = Account { program_owner: simple_transfer_id, balance: 1, @@ -630,9 +628,8 @@ fn private_pda_update_encrypts_pda_kind_with_identifier() { Program::serialize_instruction((seed, 1_u128, simple_transfer_id, false)).unwrap(), vec![ InputAccountIdentity::PrivatePdaUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), - ssk, + vpk: keys.vpk(), + random_seed: [0; 32], nsk: keys.nsk, membership_proof: commitment_set.get_proof_for(&pda_commitment).unwrap(), identifier, @@ -660,19 +657,16 @@ fn private_pda_init_identifier_mismatch_fails() { let keys = test_private_account_keys_1(); let npk = keys.npk(); let seed = PdaSeed::new([42; 32]); - let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - - let account_id = AccountId::for_private_pda(&program.id(), &seed, &npk, 5); + let account_id = AccountId::for_private_pda(&program.id(), &seed, &npk, &keys.vpk(), 5); let pre_state = AccountWithMetadata::new(Account::default(), false, account_id); let result = execute_and_prove( vec![pre_state], Program::serialize_instruction(seed).unwrap(), vec![InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret, identifier: 99, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -690,10 +684,8 @@ fn private_pda_update_identifier_mismatch_fails() { let keys = test_private_account_keys_1(); let npk = keys.npk(); let seed = PdaSeed::new([42; 32]); - let ssk = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - let simple_transfer_id = simple_transfer.id(); - let pda_id = AccountId::for_private_pda(&program.id(), &seed, &npk, 5); + let pda_id = AccountId::for_private_pda(&program.id(), &seed, &npk, &keys.vpk(), 5); let pda_account = Account { program_owner: simple_transfer_id, balance: 1, @@ -714,9 +706,8 @@ fn private_pda_update_identifier_mismatch_fails() { Program::serialize_instruction((seed, 1_u128, simple_transfer_id, false)).unwrap(), vec![ InputAccountIdentity::PrivatePdaUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), - ssk, + vpk: keys.vpk(), + random_seed: [0; 32], nsk: keys.nsk, membership_proof: commitment_set.get_proof_for(&pda_commitment).unwrap(), identifier: 99, diff --git a/lee/state_machine/src/privacy_preserving_transaction/message.rs b/lee/state_machine/src/privacy_preserving_transaction/message.rs index b2594912..73e62021 100644 --- a/lee/state_machine/src/privacy_preserving_transaction/message.rs +++ b/lee/state_machine/src/privacy_preserving_transaction/message.rs @@ -89,8 +89,8 @@ impl Message { #[cfg(test)] pub mod tests { use lee_core::{ - Commitment, EncryptionScheme, Nullifier, NullifierPublicKey, PrivateAccountKind, - SharedSecretKey, + Commitment, EncryptionScheme, EphemeralSecretKey, Nullifier, NullifierPublicKey, + PrivateAccountKind, SharedSecretKey, account::{Account, AccountId, Nonce}, encryption::ViewingPublicKey, program::{BlockValidityWindow, TimestampValidityWindow}, @@ -109,6 +109,7 @@ pub mod tests { let npk1 = NullifierPublicKey::from(&nsk1); let npk2 = NullifierPublicKey::from(&nsk2); + let vpk = ViewingPublicKey::from_seed(&[7; 32], &[8; 32]); let public_account_ids = vec![AccountId::new([1; 32])]; @@ -118,10 +119,10 @@ pub mod tests { let encrypted_private_post_states = Vec::new(); - let account_id2 = lee_core::account::AccountId::for_regular_private_account(&npk2, 0); + let account_id2 = lee_core::account::AccountId::for_regular_private_account(&npk2, &vpk, 0); let new_commitments = vec![Commitment::new(&account_id2, &account2)]; - let account_id1 = lee_core::account::AccountId::for_regular_private_account(&npk1, 0); + let account_id1 = lee_core::account::AccountId::for_regular_private_account(&npk1, &vpk, 0); let old_commitment = Commitment::new(&account_id1, &account1); let new_nullifiers = vec![( Nullifier::for_account_update(&old_commitment, &nsk1), @@ -197,9 +198,10 @@ pub mod tests { let npk = NullifierPublicKey::from(&[1; 32]); let vpk = ViewingPublicKey::from_seed(&[2_u8; 32], &[3_u8; 32]); let account = Account::default(); - let account_id = lee_core::account::AccountId::for_regular_private_account(&npk, 0); + let account_id = lee_core::account::AccountId::for_regular_private_account(&npk, &vpk, 0); let commitment = Commitment::new(&account_id, &account); - let (shared_secret, epk) = SharedSecretKey::encapsulate_deterministic(&vpk, &[0_u8; 32], 0); + let (shared_secret, epk) = + SharedSecretKey::encapsulate_deterministic(&vpk, &EphemeralSecretKey([0_u8; 32])); let ciphertext = EncryptionScheme::encrypt( &account, &PrivateAccountKind::Regular(0), diff --git a/lee/state_machine/src/program_deployment_transaction/message.rs b/lee/state_machine/src/program_deployment_transaction/message.rs index 866399e8..34c6a806 100644 --- a/lee/state_machine/src/program_deployment_transaction/message.rs +++ b/lee/state_machine/src/program_deployment_transaction/message.rs @@ -1,10 +1,18 @@ use borsh::{BorshDeserialize, BorshSerialize}; -#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] +#[derive(Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub struct Message { pub(crate) bytecode: Vec, } +impl std::fmt::Debug for Message { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Message") + .field("bytecode", &format_args!("<{} bytes>", self.bytecode.len())) + .finish() + } +} + impl Message { #[must_use] pub const fn new(bytecode: Vec) -> Self { diff --git a/lee/state_machine/src/state/tests/changer_claimer.rs b/lee/state_machine/src/state/tests/changer_claimer.rs index f9ddce25..9a581c63 100644 --- a/lee/state_machine/src/state/tests/changer_claimer.rs +++ b/lee/state_machine/src/state/tests/changer_claimer.rs @@ -60,8 +60,11 @@ fn public_changer_claimer_data_change_no_claim_fails() { fn private_changer_claimer_no_data_change_no_claim_succeeds() { let program = crate::test_methods::changer_claimer(); let sender_keys = test_private_account_keys_1(); - let private_account = - AccountWithMetadata::new(Account::default(), true, (&sender_keys.npk(), 0)); + let private_account = AccountWithMetadata::new( + Account::default(), + true, + (&sender_keys.npk(), &sender_keys.vpk(), 0), + ); // Don't change data (None) and don't claim (false) let instruction: (Option>, bool) = (None, false); @@ -69,12 +72,8 @@ fn private_changer_claimer_no_data_change_no_claim_succeeds() { vec![private_account], Program::serialize_instruction(instruction).unwrap(), vec![InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0).0, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (0, vec![]), identifier: 0, @@ -90,8 +89,11 @@ fn private_changer_claimer_no_data_change_no_claim_succeeds() { fn private_changer_claimer_data_change_no_claim_fails() { let program = crate::test_methods::changer_claimer(); let sender_keys = test_private_account_keys_1(); - let private_account = - AccountWithMetadata::new(Account::default(), true, (&sender_keys.npk(), 0)); + let private_account = AccountWithMetadata::new( + Account::default(), + true, + (&sender_keys.npk(), &sender_keys.vpk(), 0), + ); // Change data but don't claim (false) - should fail let new_data = vec![1, 2, 3, 4, 5]; let instruction: (Option>, bool) = (Some(new_data), false); @@ -100,12 +102,8 @@ fn private_changer_claimer_data_change_no_claim_fails() { vec![private_account], Program::serialize_instruction(instruction).unwrap(), vec![InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0).0, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (0, vec![]), identifier: 0, diff --git a/lee/state_machine/src/state/tests/circuit.rs b/lee/state_machine/src/state/tests/circuit.rs index 0860e98b..c13a7370 100644 --- a/lee/state_machine/src/state/tests/circuit.rs +++ b/lee/state_machine/src/state/tests/circuit.rs @@ -45,10 +45,13 @@ fn circuit_fails_if_invalid_auth_keys_are_provided() { ..Account::default() }, true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), + ); + let private_account_2 = AccountWithMetadata::new( + Account::default(), + false, + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), ); - let private_account_2 = - AccountWithMetadata::new(Account::default(), false, (&recipient_keys.npk(), 0)); // Setting the recipient nsk to authorize the sender. // This should be set to the sender private account in a normal circumstance. @@ -59,32 +62,18 @@ fn circuit_fails_if_invalid_auth_keys_are_provided() { Program::serialize_instruction(10_u128).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0) - .0, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: recipient_keys.nsk, membership_proof: (0, vec![]), identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: SharedSecretKey::encapsulate_deterministic( - &recipient_keys.vpk(), - &[0_u8; 32], - 0, - ) - .0, - commitment_root: DUMMY_COMMITMENT_HASH, identifier: 0, + commitment_root: DUMMY_COMMITMENT_HASH, }, ], &program.into(), @@ -105,7 +94,7 @@ fn circuit_should_fail_if_new_private_account_with_non_default_balance_is_provid ..Account::default() }, true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), ); let private_account_2 = AccountWithMetadata::new( Account { @@ -114,7 +103,7 @@ fn circuit_should_fail_if_new_private_account_with_non_default_balance_is_provid ..Account::default() }, false, - (&recipient_keys.npk(), 0), + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), ); let result = execute_and_prove( @@ -122,32 +111,18 @@ fn circuit_should_fail_if_new_private_account_with_non_default_balance_is_provid Program::serialize_instruction(10_u128).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0) - .0, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (0, vec![]), identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: SharedSecretKey::encapsulate_deterministic( - &recipient_keys.vpk(), - &[0_u8; 32], - 0, - ) - .0, - commitment_root: DUMMY_COMMITMENT_HASH, identifier: 0, + commitment_root: DUMMY_COMMITMENT_HASH, }, ], &program.into(), @@ -168,7 +143,7 @@ fn circuit_should_fail_if_new_private_account_with_non_default_program_owner_is_ ..Account::default() }, true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), ); let private_account_2 = AccountWithMetadata::new( Account { @@ -177,7 +152,7 @@ fn circuit_should_fail_if_new_private_account_with_non_default_program_owner_is_ ..Account::default() }, false, - (&recipient_keys.npk(), 0), + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), ); let result = execute_and_prove( @@ -185,32 +160,18 @@ fn circuit_should_fail_if_new_private_account_with_non_default_program_owner_is_ Program::serialize_instruction(10_u128).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0) - .0, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (0, vec![]), identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: SharedSecretKey::encapsulate_deterministic( - &recipient_keys.vpk(), - &[0_u8; 32], - 0, - ) - .0, - commitment_root: DUMMY_COMMITMENT_HASH, identifier: 0, + commitment_root: DUMMY_COMMITMENT_HASH, }, ], &program.into(), @@ -231,7 +192,7 @@ fn circuit_should_fail_if_new_private_account_with_non_default_data_is_provided( ..Account::default() }, true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), ); let private_account_2 = AccountWithMetadata::new( Account { @@ -240,7 +201,7 @@ fn circuit_should_fail_if_new_private_account_with_non_default_data_is_provided( ..Account::default() }, false, - (&recipient_keys.npk(), 0), + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), ); let result = execute_and_prove( @@ -248,32 +209,18 @@ fn circuit_should_fail_if_new_private_account_with_non_default_data_is_provided( Program::serialize_instruction(10_u128).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0) - .0, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (0, vec![]), identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: SharedSecretKey::encapsulate_deterministic( - &recipient_keys.vpk(), - &[0_u8; 32], - 0, - ) - .0, - commitment_root: DUMMY_COMMITMENT_HASH, identifier: 0, + commitment_root: DUMMY_COMMITMENT_HASH, }, ], &program.into(), @@ -294,7 +241,7 @@ fn circuit_should_fail_if_new_private_account_with_non_default_nonce_is_provided ..Account::default() }, true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), ); let private_account_2 = AccountWithMetadata::new( Account { @@ -303,7 +250,7 @@ fn circuit_should_fail_if_new_private_account_with_non_default_nonce_is_provided ..Account::default() }, false, - (&recipient_keys.npk(), 0), + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), ); let result = execute_and_prove( @@ -311,32 +258,18 @@ fn circuit_should_fail_if_new_private_account_with_non_default_nonce_is_provided Program::serialize_instruction(10_u128).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0) - .0, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (0, vec![]), identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: SharedSecretKey::encapsulate_deterministic( - &recipient_keys.vpk(), - &[0_u8; 32], - 0, - ) - .0, - commitment_root: DUMMY_COMMITMENT_HASH, identifier: 0, + commitment_root: DUMMY_COMMITMENT_HASH, }, ], &program.into(), @@ -358,13 +291,13 @@ fn circuit_should_fail_if_new_private_account_is_provided_with_default_values_bu ..Account::default() }, true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), ); let private_account_2 = AccountWithMetadata::new( Account::default(), // This should be set to false in normal circumstances true, - (&recipient_keys.npk(), 0), + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), ); let result = execute_and_prove( @@ -372,32 +305,18 @@ fn circuit_should_fail_if_new_private_account_is_provided_with_default_values_bu Program::serialize_instruction(10_u128).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0) - .0, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (0, vec![]), identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: SharedSecretKey::encapsulate_deterministic( - &recipient_keys.vpk(), - &[0_u8; 32], - 0, - ) - .0, - commitment_root: DUMMY_COMMITMENT_HASH, identifier: 0, + commitment_root: DUMMY_COMMITMENT_HASH, }, ], &program.into(), @@ -415,7 +334,6 @@ fn private_pda_without_binding_fails() { let program = crate::test_methods::simple_balance_transfer(); let keys = test_private_account_keys_1(); let npk = keys.npk(); - let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; let public_account_1 = AccountWithMetadata::new( Account { program_owner: program.id(), @@ -434,10 +352,9 @@ fn private_pda_without_binding_fails() { vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret, identifier: u128::MAX, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -460,19 +377,17 @@ fn private_pda_claim_succeeds() { let keys = test_private_account_keys_1(); let npk = keys.npk(); let seed = PdaSeed::new([42; 32]); - let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - let account_id = AccountId::for_private_pda(&program.id(), &seed, &npk, u128::MAX); + let account_id = AccountId::for_private_pda(&program.id(), &seed, &npk, &keys.vpk(), u128::MAX); let pre_state = AccountWithMetadata::new(Account::default(), false, account_id); let result = execute_and_prove( vec![pre_state], Program::serialize_instruction(seed).unwrap(), vec![InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret, identifier: u128::MAX, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -500,22 +415,21 @@ fn private_pda_npk_mismatch_fails() { let npk_a = keys_a.npk(); let npk_b = keys_b.npk(); let seed = PdaSeed::new([42; 32]); - let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys_b.vpk(), &[0_u8; 32], 0).0; // `account_id` is derived from `npk_a`, but `npk_b` is supplied for this pre_state. // `AccountId::for_private_pda(program, seed, npk_b) != account_id`, so the claim check in // the circuit must reject. - let account_id = AccountId::for_private_pda(&program.id(), &seed, &npk_a, u128::MAX); + let account_id = + AccountId::for_private_pda(&program.id(), &seed, &npk_a, &keys_a.vpk(), u128::MAX); let pre_state = AccountWithMetadata::new(Account::default(), false, account_id); let result = execute_and_prove( vec![pre_state], Program::serialize_instruction(seed).unwrap(), vec![InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk_b, &keys_b.vpk()), + vpk: keys_b.vpk(), + random_seed: [0; 32], npk: npk_b, - ssk: shared_secret, identifier: u128::MAX, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -538,9 +452,9 @@ fn caller_pda_seeds_authorize_private_pda_for_callee() { let keys = test_private_account_keys_1(); let npk = keys.npk(); let seed = PdaSeed::new([77; 32]); - let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - let account_id = AccountId::for_private_pda(&delegator.id(), &seed, &npk, u128::MAX); + let account_id = + AccountId::for_private_pda(&delegator.id(), &seed, &npk, &keys.vpk(), u128::MAX); let pre_state = AccountWithMetadata::new(Account::default(), false, account_id); let callee_id = callee.id(); @@ -550,10 +464,9 @@ fn caller_pda_seeds_authorize_private_pda_for_callee() { vec![pre_state], Program::serialize_instruction((seed, seed, callee_id)).unwrap(), vec![InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret, identifier: u128::MAX, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -579,9 +492,9 @@ fn caller_pda_seeds_with_wrong_seed_rejects_private_pda_for_callee() { let npk = keys.npk(); let claim_seed = PdaSeed::new([77; 32]); let wrong_delegated_seed = PdaSeed::new([88; 32]); - let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; - let account_id = AccountId::for_private_pda(&delegator.id(), &claim_seed, &npk, u128::MAX); + let account_id = + AccountId::for_private_pda(&delegator.id(), &claim_seed, &npk, &keys.vpk(), u128::MAX); let pre_state = AccountWithMetadata::new(Account::default(), false, account_id); let callee_id = callee.id(); @@ -591,10 +504,9 @@ fn caller_pda_seeds_with_wrong_seed_rejects_private_pda_for_callee() { vec![pre_state], Program::serialize_instruction((claim_seed, wrong_delegated_seed, callee_id)).unwrap(), vec![InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret, identifier: u128::MAX, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -619,11 +531,21 @@ fn two_private_pda_claims_under_same_seed_are_rejected() { let keys_a = test_private_account_keys_1(); let keys_b = test_private_account_keys_2(); let seed = PdaSeed::new([55; 32]); - let shared_a = SharedSecretKey::encapsulate_deterministic(&keys_a.vpk(), &[0_u8; 32], 0).0; - let shared_b = SharedSecretKey::encapsulate_deterministic(&keys_b.vpk(), &[0_u8; 32], 0).0; - let account_a = AccountId::for_private_pda(&program.id(), &seed, &keys_a.npk(), u128::MAX); - let account_b = AccountId::for_private_pda(&program.id(), &seed, &keys_b.npk(), u128::MAX); + let account_a = AccountId::for_private_pda( + &program.id(), + &seed, + &keys_a.npk(), + &keys_a.vpk(), + u128::MAX, + ); + let account_b = AccountId::for_private_pda( + &program.id(), + &seed, + &keys_b.npk(), + &keys_b.vpk(), + u128::MAX, + ); let pre_a = AccountWithMetadata::new(Account::default(), false, account_a); let pre_b = AccountWithMetadata::new(Account::default(), false, account_b); @@ -633,19 +555,17 @@ fn two_private_pda_claims_under_same_seed_are_rejected() { Program::serialize_instruction(seed).unwrap(), vec![ InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&keys_a.npk(), &keys_a.vpk()), + vpk: keys_a.vpk(), + random_seed: [0; 32], npk: keys_a.npk(), - ssk: shared_a, identifier: u128::MAX, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, }, InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&keys_b.npk(), &keys_b.vpk()), + vpk: keys_b.vpk(), + random_seed: [0; 32], npk: keys_b.npk(), - ssk: shared_b, identifier: u128::MAX, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -667,12 +587,11 @@ fn private_pda_top_level_reuse_rejected_by_binding_check() { let program = crate::test_methods::noop(); let keys = test_private_account_keys_1(); let npk = keys.npk(); - let shared_secret = SharedSecretKey::encapsulate_deterministic(&keys.vpk(), &[0_u8; 32], 0).0; let seed = PdaSeed::new([99; 32]); // Simulate a previously-claimed private PDA: program_owner != DEFAULT, is_authorized = // true, account_id derived via the private formula. - let account_id = AccountId::for_private_pda(&program.id(), &seed, &npk, u128::MAX); + let account_id = AccountId::for_private_pda(&program.id(), &seed, &npk, &keys.vpk(), u128::MAX); let owned_pre_state = AccountWithMetadata::new( Account { program_owner: program.id(), @@ -686,10 +605,9 @@ fn private_pda_top_level_reuse_rejected_by_binding_check() { vec![owned_pre_state], Program::serialize_instruction(()).unwrap(), vec![InputAccountIdentity::PrivatePdaInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag(&npk, &keys.vpk()), + vpk: keys.vpk(), + random_seed: [0; 32], npk, - ssk: shared_secret, identifier: u128::MAX, commitment_root: DUMMY_COMMITMENT_HASH, seed: None, @@ -766,33 +684,23 @@ fn circuit_should_fail_if_there_are_repeated_ids() { ..Account::default() }, true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), ); - let shared_secret = - SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0).0; let result = execute_and_prove( vec![private_account_1.clone(), private_account_1], Program::serialize_instruction(100_u128).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: shared_secret, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (1, vec![]), identifier: 0, }, InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: shared_secret, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: (1, vec![]), identifier: 0, @@ -812,14 +720,15 @@ fn private_authorized_uninitialized_account() { let private_keys = test_private_account_keys_1(); // Create an authorized private account with default values (new account being initialized) - let authorized_account = - AccountWithMetadata::new(Account::default(), true, (&private_keys.npk(), 0)); + let authorized_account = AccountWithMetadata::new( + Account::default(), + true, + (&private_keys.npk(), &private_keys.vpk(), 0), + ); let program = crate::test_methods::simple_balance_transfer(); // Set up parameters for the new account - let (shared_secret, epk) = - SharedSecretKey::encapsulate_deterministic(&private_keys.vpk(), &[0_u8; 32], 0); let instruction: u128 = 0; @@ -828,12 +737,8 @@ fn private_authorized_uninitialized_account() { vec![authorized_account], Program::serialize_instruction(instruction).unwrap(), vec![InputAccountIdentity::PrivateAuthorizedInit { - epk, - view_tag: EncryptedAccountData::compute_view_tag( - &private_keys.npk(), - &private_keys.vpk(), - ), - ssk: shared_secret, + vpk: private_keys.vpk(), + random_seed: [0; 32], nsk: private_keys.nsk, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, @@ -851,7 +756,8 @@ fn private_authorized_uninitialized_account() { let result = state.transition_from_privacy_preserving_transaction(&tx, 1, 0); assert!(result.is_ok()); - let account_id = AccountId::for_regular_private_account(&private_keys.npk(), 0); + let account_id = + AccountId::for_regular_private_account(&private_keys.npk(), &private_keys.vpk(), 0); let nullifier = Nullifier::for_account_initialization(&account_id); assert!(state.private_state.1.contains(&nullifier)); } @@ -865,24 +771,21 @@ fn private_unauthorized_uninitialized_account_can_still_be_claimed() { // especially PDAs. Private PDAs are not useful in practice because there is no way to // operate them without the corresponding private keys, so unauthorized private claiming // remains allowed. - let unauthorized_account = - AccountWithMetadata::new(Account::default(), false, (&private_keys.npk(), 0)); + let unauthorized_account = AccountWithMetadata::new( + Account::default(), + false, + (&private_keys.npk(), &private_keys.vpk(), 0), + ); let program = crate::test_methods::claimer(); - let (shared_secret, epk) = - SharedSecretKey::encapsulate_deterministic(&private_keys.vpk(), &[0_u8; 32], 0); let (output, proof) = execute_and_prove( vec![unauthorized_account], Program::serialize_instruction(()).unwrap(), vec![InputAccountIdentity::PrivateUnauthorized { - epk, - view_tag: EncryptedAccountData::compute_view_tag( - &private_keys.npk(), - &private_keys.vpk(), - ), + vpk: private_keys.vpk(), + random_seed: [0; 32], npk: private_keys.npk(), - ssk: shared_secret, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }], @@ -899,7 +802,8 @@ fn private_unauthorized_uninitialized_account_can_still_be_claimed() { .transition_from_privacy_preserving_transaction(&tx, 1, 0) .unwrap(); - let account_id = AccountId::for_regular_private_account(&private_keys.npk(), 0); + let account_id = + AccountId::for_regular_private_account(&private_keys.npk(), &private_keys.vpk(), 0); let nullifier = Nullifier::for_account_initialization(&account_id); assert!(state.private_state.1.contains(&nullifier)); } @@ -912,14 +816,15 @@ fn private_account_claimed_then_used_without_init_flag_should_fail() { let private_keys = test_private_account_keys_1(); // Step 1: Create a new private account with authorization - let authorized_account = - AccountWithMetadata::new(Account::default(), true, (&private_keys.npk(), 0)); + let authorized_account = AccountWithMetadata::new( + Account::default(), + true, + (&private_keys.npk(), &private_keys.vpk(), 0), + ); let claimer_program = crate::test_methods::claimer(); // Set up parameters for claiming the new account - let (shared_secret, epk) = - SharedSecretKey::encapsulate_deterministic(&private_keys.vpk(), &[0_u8; 32], 0); let instruction = (); @@ -928,12 +833,8 @@ fn private_account_claimed_then_used_without_init_flag_should_fail() { vec![authorized_account.clone()], Program::serialize_instruction(instruction).unwrap(), vec![InputAccountIdentity::PrivateAuthorizedInit { - epk, - view_tag: EncryptedAccountData::compute_view_tag( - &private_keys.npk(), - &private_keys.vpk(), - ), - ssk: shared_secret, + vpk: private_keys.vpk(), + random_seed: [0; 32], nsk: private_keys.nsk, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, @@ -955,7 +856,8 @@ fn private_account_claimed_then_used_without_init_flag_should_fail() { ); // Verify the account is now initialized (nullifier exists) - let account_id = AccountId::for_regular_private_account(&private_keys.npk(), 0); + let account_id = + AccountId::for_regular_private_account(&private_keys.npk(), &private_keys.vpk(), 0); let nullifier = Nullifier::for_account_initialization(&account_id); assert!(state.private_state.1.contains(&nullifier)); @@ -967,20 +869,14 @@ fn private_account_claimed_then_used_without_init_flag_should_fail() { }; let noop_program = crate::test_methods::noop(); - let shared_secret2 = - SharedSecretKey::encapsulate_deterministic(&private_keys.vpk(), &[0_u8; 32], 0).0; // Step 3: Try to execute noop program with authentication but without initialization let res = execute_and_prove( vec![account_metadata], Program::serialize_instruction(()).unwrap(), vec![InputAccountIdentity::PrivateAuthorizedInit { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &private_keys.npk(), - &private_keys.vpk(), - ), - ssk: shared_secret2, + vpk: private_keys.vpk(), + random_seed: [0; 32], nsk: private_keys.nsk, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, @@ -1010,8 +906,10 @@ fn two_private_pda_family_members_receive_and_spend() { ); let funder_id = funder_keys.account_id(); - let alice_pda_0_id = AccountId::for_private_pda(&proxy_id, &seed, &alice_npk, 0); - let alice_pda_1_id = AccountId::for_private_pda(&proxy_id, &seed, &alice_npk, 1); + let alice_pda_0_id = + AccountId::for_private_pda(&proxy_id, &seed, &alice_npk, &alice_keys.vpk(), 0); + let alice_pda_1_id = + AccountId::for_private_pda(&proxy_id, &seed, &alice_npk, &alice_keys.vpk(), 1); let recipient_id = test_public_account_keys_2().account_id(); let recipient_signing_key = test_public_account_keys_2().signing_key; @@ -1031,11 +929,6 @@ fn two_private_pda_family_members_receive_and_spend() { ..Account::default() }; - let (alice_shared_0, alice_epk_0) = - SharedSecretKey::encapsulate_deterministic(&alice_keys.vpk(), &[0_u8; 32], 0); - let (alice_shared_1, alice_epk_1) = - SharedSecretKey::encapsulate_deterministic(&alice_keys.vpk(), &[0_u8; 32], 1); - // Fund alice_pda_0 via authenticated_transfer directly. { let funder_account = state.get_account_by_id(funder_id); @@ -1049,10 +942,9 @@ fn two_private_pda_family_members_receive_and_spend() { vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivatePdaInit { - epk: alice_epk_0.clone(), - view_tag: EncryptedAccountData::compute_view_tag(&alice_npk, &alice_keys.vpk()), + vpk: alice_keys.vpk(), + random_seed: [0; 32], npk: alice_npk, - ssk: alice_shared_0, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, seed: Some((seed, proxy_id)), @@ -1086,10 +978,9 @@ fn two_private_pda_family_members_receive_and_spend() { vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivatePdaInit { - epk: alice_epk_1.clone(), - view_tag: EncryptedAccountData::compute_view_tag(&alice_npk, &alice_keys.vpk()), + vpk: alice_keys.vpk(), + random_seed: [0; 32], npk: alice_npk, - ssk: alice_shared_1, identifier: 1, commitment_root: DUMMY_COMMITMENT_HASH, seed: Some((seed, proxy_id)), @@ -1127,9 +1018,8 @@ fn two_private_pda_family_members_receive_and_spend() { Program::serialize_instruction((seed, amount, simple_transfer_id)).unwrap(), vec![ InputAccountIdentity::PrivatePdaUpdate { - epk: alice_epk_0, - view_tag: EncryptedAccountData::compute_view_tag(&alice_npk, &alice_keys.vpk()), - ssk: alice_shared_0, + vpk: alice_keys.vpk(), + random_seed: [0; 32], nsk: alice_keys.nsk, membership_proof: state .get_proof_for_commitment(&commitment_pda_0) @@ -1165,9 +1055,8 @@ fn two_private_pda_family_members_receive_and_spend() { Program::serialize_instruction((seed, amount, simple_transfer_id)).unwrap(), vec![ InputAccountIdentity::PrivatePdaUpdate { - epk: alice_epk_1, - view_tag: EncryptedAccountData::compute_view_tag(&alice_npk, &alice_keys.vpk()), - ssk: alice_shared_1, + vpk: alice_keys.vpk(), + random_seed: [0; 32], nsk: alice_keys.nsk, membership_proof: state .get_proof_for_commitment(&commitment_pda_1) @@ -1205,7 +1094,6 @@ fn two_private_pda_family_members_receive_and_spend() { }; let commitment_pda_1_after_spend = Commitment::new(&alice_pda_1_id, &alice_pda_1_account_after_spend); - let alice_shared_1_refund = SharedSecretKey([12; 32]); { let recipient_account = state.get_account_by_id(recipient_id); let recipient_nonce = recipient_account.nonce; @@ -1218,10 +1106,9 @@ fn two_private_pda_family_members_receive_and_spend() { vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivatePdaUpdate { - epk: EphemeralPublicKey(vec![12_u8; ML_KEM_768_CIPHERTEXT_LEN]), - view_tag: EncryptedAccountData::compute_view_tag(&alice_npk, &alice_keys.vpk()), + vpk: alice_keys.vpk(), + random_seed: [0; 32], nsk: alice_keys.nsk, - ssk: alice_shared_1_refund, membership_proof: state .get_proof_for_commitment(&commitment_pda_1_after_spend) .expect("pda_1 after spend must be in state"), diff --git a/lee/state_machine/src/state/tests/claiming.rs b/lee/state_machine/src/state/tests/claiming.rs index aba5c447..acfe9834 100644 --- a/lee/state_machine/src/state/tests/claiming.rs +++ b/lee/state_machine/src/state/tests/claiming.rs @@ -303,19 +303,21 @@ fn authorized_public_account_claiming_succeeds_when_executed_privately() { balance: 100, ..Account::default() }; - let sender_account_id = AccountId::for_regular_private_account(&sender_keys.npk(), 0); + let sender_account_id = + AccountId::for_regular_private_account(&sender_keys.npk(), &sender_keys.vpk(), 0); let sender_commitment = Commitment::new(&sender_account_id, &sender_private_account); let sender_init_nullifier = Nullifier::for_account_initialization(&sender_account_id); let mut state = V03State::new().with_private_accounts([(sender_commitment.clone(), sender_init_nullifier)]); - let sender_pre = - AccountWithMetadata::new(sender_private_account, true, (&sender_keys.npk(), 0)); + let sender_pre = AccountWithMetadata::new( + sender_private_account, + true, + (&sender_keys.npk(), &sender_keys.vpk(), 0), + ); let recipient_private_key = PrivateKey::try_new([2; 32]).unwrap(); let recipient_account_id = AccountId::from(&PublicKey::new_from_private_key(&recipient_private_key)); let recipient_pre = AccountWithMetadata::new(Account::default(), true, recipient_account_id); - let (shared_secret, epk) = - SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0); let balance = 37; @@ -324,12 +326,8 @@ fn authorized_public_account_claiming_succeeds_when_executed_privately() { Program::serialize_instruction(balance).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk, - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: shared_secret, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: state .get_proof_for_commitment(&sender_commitment) @@ -383,7 +381,7 @@ fn private_chained_call(number_of_calls: u32) { ..Account::default() }, true, - (&from_keys.npk(), 0), + (&from_keys.npk(), &from_keys.vpk(), 0), ); let to_account = AccountWithMetadata::new( Account { @@ -391,11 +389,12 @@ fn private_chained_call(number_of_calls: u32) { ..Account::default() }, true, - (&to_keys.npk(), 0), + (&to_keys.npk(), &to_keys.vpk(), 0), ); - let from_account_id = AccountId::for_regular_private_account(&from_keys.npk(), 0); - let to_account_id = AccountId::for_regular_private_account(&to_keys.npk(), 0); + let from_account_id = + AccountId::for_regular_private_account(&from_keys.npk(), &from_keys.vpk(), 0); + let to_account_id = AccountId::for_regular_private_account(&to_keys.npk(), &to_keys.vpk(), 0); let from_commitment = Commitment::new(&from_account_id, &from_account.account); let to_commitment = Commitment::new(&to_account_id, &to_account.account); let from_init_nullifier = Nullifier::for_account_initialization(&from_account_id); @@ -414,12 +413,6 @@ fn private_chained_call(number_of_calls: u32) { None, ); - let (from_ss, from_epk) = - SharedSecretKey::encapsulate_deterministic(&from_keys.vpk(), &[0_u8; 32], 0); - - let (to_ss, to_epk) = - SharedSecretKey::encapsulate_deterministic(&to_keys.vpk(), &[0_u8; 32], 1); - let mut dependencies = HashMap::new(); dependencies.insert(simple_transfers.id(), simple_transfers); @@ -448,9 +441,8 @@ fn private_chained_call(number_of_calls: u32) { Program::serialize_instruction(instruction).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: to_epk, - view_tag: EncryptedAccountData::compute_view_tag(&to_keys.npk(), &to_keys.vpk()), - ssk: to_ss, + vpk: from_keys.vpk(), + random_seed: [0; 32], nsk: from_keys.nsk, membership_proof: state .get_proof_for_commitment(&from_commitment) @@ -458,12 +450,8 @@ fn private_chained_call(number_of_calls: u32) { identifier: 0, }, InputAccountIdentity::PrivateAuthorizedUpdate { - epk: from_epk, - view_tag: EncryptedAccountData::compute_view_tag( - &from_keys.npk(), - &from_keys.vpk(), - ), - ssk: from_ss, + vpk: to_keys.vpk(), + random_seed: [0; 32], nsk: to_keys.nsk, membership_proof: state .get_proof_for_commitment(&to_commitment) diff --git a/lee/state_machine/src/state/tests/genesis.rs b/lee/state_machine/src/state/tests/genesis.rs index d782f325..f67628ee 100644 --- a/lee/state_machine/src/state/tests/genesis.rs +++ b/lee/state_machine/src/state/tests/genesis.rs @@ -43,8 +43,8 @@ fn new_includes_nullifiers_for_private_accounts() { ..Account::default() }; - let account_id1 = AccountId::for_regular_private_account(&keys1.npk(), 0); - let account_id2 = AccountId::for_regular_private_account(&keys2.npk(), 0); + let account_id1 = AccountId::for_regular_private_account(&keys1.npk(), &keys1.vpk(), 0); + let account_id2 = AccountId::for_regular_private_account(&keys2.npk(), &keys2.vpk(), 0); let init_commitment1 = Commitment::new(&account_id1, &account); let init_commitment2 = Commitment::new(&account_id2, &account); diff --git a/lee/state_machine/src/state/tests/mod.rs b/lee/state_machine/src/state/tests/mod.rs index f6b56be6..55a99ead 100644 --- a/lee/state_machine/src/state/tests/mod.rs +++ b/lee/state_machine/src/state/tests/mod.rs @@ -7,10 +7,10 @@ use std::collections::HashMap; use lee_core::{ - BlockId, Commitment, DUMMY_COMMITMENT_HASH, EncryptedAccountData, InputAccountIdentity, - Nullifier, NullifierPublicKey, NullifierSecretKey, SharedSecretKey, Timestamp, + BlockId, Commitment, DUMMY_COMMITMENT_HASH, InputAccountIdentity, Nullifier, + NullifierPublicKey, NullifierSecretKey, Timestamp, account::{Account, AccountId, AccountWithMetadata, Nonce, data::Data}, - encryption::{EphemeralPublicKey, ML_KEM_768_CIPHERTEXT_LEN, ViewingPublicKey}, + encryption::ViewingPublicKey, program::{ BlockValidityWindow, ExecutionValidationError, MAX_NUMBER_CHAINED_CALLS, PdaSeed, ProgramId, TimestampValidityWindow, WrappedBalanceSum, @@ -118,7 +118,7 @@ impl V03State { #[must_use] pub fn with_private_account(mut self, keys: &TestPrivateKeys, account: &Account) -> Self { - let account_id = AccountId::for_regular_private_account(&keys.npk(), 0); + let account_id = AccountId::for_regular_private_account(&keys.npk(), &keys.vpk(), 0); let commitment = Commitment::new(&account_id, account); self.private_state.0.extend(&[commitment]); self @@ -267,10 +267,11 @@ fn shielded_balance_transfer_for_tests( let sender_nonce = sender.account.nonce; - let recipient = AccountWithMetadata::new(Account::default(), false, (&recipient_keys.npk(), 0)); - - let (shared_secret, epk) = - SharedSecretKey::encapsulate_deterministic(&recipient_keys.vpk(), &[0_u8; 32], 0); + let recipient = AccountWithMetadata::new( + Account::default(), + false, + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), + ); let (output, proof) = crate::privacy_preserving_transaction::circuit::execute_and_prove( vec![sender, recipient], @@ -278,13 +279,9 @@ fn shielded_balance_transfer_for_tests( vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivateUnauthorized { - epk, - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: shared_secret, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }, @@ -312,33 +309,27 @@ fn private_balance_transfer_for_tests( state: &V03State, ) -> PrivacyPreservingTransaction { let program = crate::test_methods::simple_balance_transfer(); - let sender_account_id = AccountId::for_regular_private_account(&sender_keys.npk(), 0); + let sender_account_id = + AccountId::for_regular_private_account(&sender_keys.npk(), &sender_keys.vpk(), 0); let sender_commitment = Commitment::new(&sender_account_id, sender_private_account); let sender_pre = AccountWithMetadata::new( sender_private_account.clone(), true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), + ); + let recipient_pre = AccountWithMetadata::new( + Account::default(), + false, + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), ); - let recipient_pre = - AccountWithMetadata::new(Account::default(), false, (&recipient_keys.npk(), 0)); - - let (shared_secret_1, epk_1) = - SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0); - - let (shared_secret_2, epk_2) = - SharedSecretKey::encapsulate_deterministic(&recipient_keys.vpk(), &[0_u8; 32], 1); let (output, proof) = crate::privacy_preserving_transaction::circuit::execute_and_prove( vec![sender_pre, recipient_pre], Program::serialize_instruction(balance_to_move).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: epk_1, - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: shared_secret_1, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: state .get_proof_for_commitment(&sender_commitment) @@ -346,13 +337,9 @@ fn private_balance_transfer_for_tests( identifier: 0, }, InputAccountIdentity::PrivateUnauthorized { - epk: epk_2, - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), + vpk: recipient_keys.vpk(), + random_seed: [0; 32], npk: recipient_keys.npk(), - ssk: shared_secret_2, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }, @@ -376,12 +363,13 @@ fn deshielded_balance_transfer_for_tests( state: &V03State, ) -> PrivacyPreservingTransaction { let program = crate::test_methods::simple_balance_transfer(); - let sender_account_id = AccountId::for_regular_private_account(&sender_keys.npk(), 0); + let sender_account_id = + AccountId::for_regular_private_account(&sender_keys.npk(), &sender_keys.vpk(), 0); let sender_commitment = Commitment::new(&sender_account_id, sender_private_account); let sender_pre = AccountWithMetadata::new( sender_private_account.clone(), true, - (&sender_keys.npk(), 0), + (&sender_keys.npk(), &sender_keys.vpk(), 0), ); let recipient_pre = AccountWithMetadata::new( state.get_account_by_id(*recipient_account_id), @@ -389,20 +377,13 @@ fn deshielded_balance_transfer_for_tests( *recipient_account_id, ); - let (shared_secret, epk) = - SharedSecretKey::encapsulate_deterministic(&sender_keys.vpk(), &[0_u8; 32], 0); - let (output, proof) = crate::privacy_preserving_transaction::circuit::execute_and_prove( vec![sender_pre, recipient_pre], Program::serialize_instruction(balance_to_move).unwrap(), vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk, - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.npk(), - &sender_keys.vpk(), - ), - ssk: shared_secret, + vpk: sender_keys.vpk(), + random_seed: [0; 32], nsk: sender_keys.nsk, membership_proof: state .get_proof_for_commitment(&sender_commitment) diff --git a/lee/state_machine/src/state/tests/privacy_preserving.rs b/lee/state_machine/src/state/tests/privacy_preserving.rs index 0205b5fd..8c33e202 100644 --- a/lee/state_machine/src/state/tests/privacy_preserving.rs +++ b/lee/state_machine/src/state/tests/privacy_preserving.rs @@ -68,8 +68,10 @@ fn transition_from_privacy_preserving_transaction_private() { &state, ); - let sender_account_id = AccountId::for_regular_private_account(&sender_keys.npk(), 0); - let recipient_account_id = AccountId::for_regular_private_account(&recipient_keys.npk(), 0); + let sender_account_id = + AccountId::for_regular_private_account(&sender_keys.npk(), &sender_keys.vpk(), 0); + let recipient_account_id = + AccountId::for_regular_private_account(&recipient_keys.npk(), &recipient_keys.vpk(), 0); let expected_new_commitment_1 = Commitment::new( &sender_account_id, &Account { @@ -203,7 +205,8 @@ fn transition_from_privacy_preserving_transaction_deshielded() { &state, ); - let sender_account_id = AccountId::for_regular_private_account(&sender_keys.npk(), 0); + let sender_account_id = + AccountId::for_regular_private_account(&sender_keys.npk(), &sender_keys.vpk(), 0); let expected_new_commitment = Commitment::new( &sender_account_id, &Account { @@ -487,10 +490,14 @@ fn malicious_authorization_changer_should_fail_in_privacy_preserving_circuit() { false, sender_keys.account_id(), ); - let recipient_account = - AccountWithMetadata::new(Account::default(), true, (&recipient_keys.npk(), 0)); + let recipient_account = AccountWithMetadata::new( + Account::default(), + true, + (&recipient_keys.npk(), &recipient_keys.vpk(), 0), + ); - let recipient_account_id = AccountId::for_regular_private_account(&recipient_keys.npk(), 0); + let recipient_account_id = + AccountId::for_regular_private_account(&recipient_keys.npk(), &recipient_keys.vpk(), 0); let recipient_commitment = Commitment::new(&recipient_account_id, &recipient_account.account); let recipient_init_nullifier = Nullifier::for_account_initialization(&recipient_account_id); let state = V03State::new() @@ -504,9 +511,6 @@ fn malicious_authorization_changer_should_fail_in_privacy_preserving_circuit() { let balance_to_transfer = 10_u128; let instruction = (balance_to_transfer, simple_transfers.id()); - let recipient = - SharedSecretKey::encapsulate_deterministic(&recipient_keys.vpk(), &[0_u8; 32], 0).0; - let mut dependencies = HashMap::new(); dependencies.insert(simple_transfers.id(), simple_transfers); let program_with_deps = ProgramWithDependencies::new(malicious_program, dependencies); @@ -518,12 +522,8 @@ fn malicious_authorization_changer_should_fail_in_privacy_preserving_circuit() { vec![ InputAccountIdentity::Public, InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(Vec::new()), - view_tag: EncryptedAccountData::compute_view_tag( - &recipient_keys.npk(), - &recipient_keys.vpk(), - ), - ssk: recipient, + vpk: recipient_keys.vpk(), + random_seed: [0; 32], nsk: recipient_keys.nsk, membership_proof: state .get_proof_for_commitment(&recipient_commitment) diff --git a/lee/state_machine/src/state/tests/validity_window.rs b/lee/state_machine/src/state/tests/validity_window.rs index 5a46bd5b..12e7c229 100644 --- a/lee/state_machine/src/state/tests/validity_window.rs +++ b/lee/state_machine/src/state/tests/validity_window.rs @@ -124,12 +124,13 @@ fn validity_window_works_in_privacy_preserving_transactions( let block_validity_window: BlockValidityWindow = validity_window.try_into().unwrap(); let validity_window_program = crate::test_methods::validity_window(); let account_keys = test_private_account_keys_1(); - let pre = AccountWithMetadata::new(Account::default(), false, (&account_keys.npk(), 0)); + let pre = AccountWithMetadata::new( + Account::default(), + false, + (&account_keys.npk(), &account_keys.vpk(), 0), + ); let mut state = V03State::new().with_test_programs(); let tx = { - let (shared_secret, epk) = - SharedSecretKey::encapsulate_deterministic(&account_keys.vpk(), &[0_u8; 32], 0); - let instruction = ( block_validity_window, TimestampValidityWindow::new_unbounded(), @@ -138,13 +139,9 @@ fn validity_window_works_in_privacy_preserving_transactions( vec![pre], Program::serialize_instruction(instruction).unwrap(), vec![InputAccountIdentity::PrivateUnauthorized { - epk, - view_tag: EncryptedAccountData::compute_view_tag( - &account_keys.npk(), - &account_keys.vpk(), - ), + vpk: account_keys.vpk(), + random_seed: [0; 32], npk: account_keys.npk(), - ssk: shared_secret, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }], @@ -192,12 +189,13 @@ fn timestamp_validity_window_works_in_privacy_preserving_transactions( let timestamp_validity_window: TimestampValidityWindow = validity_window.try_into().unwrap(); let validity_window_program = crate::test_methods::validity_window(); let account_keys = test_private_account_keys_1(); - let pre = AccountWithMetadata::new(Account::default(), false, (&account_keys.npk(), 0)); + let pre = AccountWithMetadata::new( + Account::default(), + false, + (&account_keys.npk(), &account_keys.vpk(), 0), + ); let mut state = V03State::new().with_test_programs(); let tx = { - let (shared_secret, epk) = - SharedSecretKey::encapsulate_deterministic(&account_keys.vpk(), &[0_u8; 32], 0); - let instruction = ( BlockValidityWindow::new_unbounded(), timestamp_validity_window, @@ -206,13 +204,9 @@ fn timestamp_validity_window_works_in_privacy_preserving_transactions( vec![pre], Program::serialize_instruction(instruction).unwrap(), vec![InputAccountIdentity::PrivateUnauthorized { - epk, - view_tag: EncryptedAccountData::compute_view_tag( - &account_keys.npk(), - &account_keys.vpk(), - ), + vpk: account_keys.vpk(), + random_seed: [0; 32], npk: account_keys.npk(), - ssk: shared_secret, identifier: 0, commitment_root: DUMMY_COMMITMENT_HASH, }], diff --git a/lee/state_machine/src/validated_state_diff/tests.rs b/lee/state_machine/src/validated_state_diff/tests.rs index 3cb61e7d..c471676c 100644 --- a/lee/state_machine/src/validated_state_diff/tests.rs +++ b/lee/state_machine/src/validated_state_diff/tests.rs @@ -78,7 +78,7 @@ fn public_diff_reflects_a_successful_transfer() { #[test] fn privacy_malicious_programs_cannot_drain_public_victim() { use lee_core::{ - Commitment, EncryptedAccountData, InputAccountIdentity, SharedSecretKey, + Commitment, InputAccountIdentity, account::{Account, AccountWithMetadata}, }; @@ -105,8 +105,8 @@ fn privacy_malicious_programs_cannot_drain_public_victim() { // Attacker controls a private account. let attacker_keys = test_private_account_keys_1(); - let attacker_id = AccountId::for_regular_private_account(&attacker_keys.npk(), 0); - let (attacker_ssk, attacker_epk) = SharedSecretKey::encapsulate(&attacker_keys.vpk()); + let attacker_id = + AccountId::for_regular_private_account(&attacker_keys.npk(), &attacker_keys.vpk(), 0); let victim_id = AccountId::new([20_u8; 32]); let recipient_id = AccountId::new([42_u8; 32]); @@ -165,12 +165,8 @@ fn privacy_malicious_programs_cannot_drain_public_victim() { // [2] recipient โ€” first seen in simple_balance_transfer's program_output.pre_states let account_identities = vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: attacker_epk, - view_tag: EncryptedAccountData::compute_view_tag( - &attacker_keys.npk(), - &attacker_keys.vpk(), - ), - ssk: attacker_ssk, + vpk: attacker_keys.vpk(), + random_seed: [0; 32], nsk: attacker_keys.nsk, membership_proof, identifier: 0, @@ -232,7 +228,7 @@ fn privacy_malicious_programs_cannot_drain_public_victim() { #[test] fn privacy_malicious_programs_cannot_drain_private_victim() { use lee_core::{ - Commitment, EncryptedAccountData, InputAccountIdentity, SharedSecretKey, + Commitment, InputAccountIdentity, account::{Account, AccountWithMetadata}, }; @@ -262,12 +258,13 @@ fn privacy_malicious_programs_cannot_drain_private_victim() { // Attacker controls a private account. let attacker_keys = test_private_account_keys_1(); - let attacker_id = AccountId::for_regular_private_account(&attacker_keys.npk(), 0); - let (attacker_ssk, attacker_epk) = SharedSecretKey::encapsulate(&attacker_keys.vpk()); + let attacker_id = + AccountId::for_regular_private_account(&attacker_keys.npk(), &attacker_keys.vpk(), 0); // Victim is a private account โ€” not registered in public chain state. let victim_keys = test_private_account_keys_2(); - let victim_id = AccountId::for_regular_private_account(&victim_keys.npk(), 0); + let victim_id = + AccountId::for_regular_private_account(&victim_keys.npk(), &victim_keys.vpk(), 0); let victim_balance = 5_000_u128; let recipient_id = AccountId::new([42_u8; 32]); @@ -328,12 +325,8 @@ fn privacy_malicious_programs_cannot_drain_private_victim() { // so PrivateAuthorizedUpdate is not an option. let account_identities = vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: attacker_epk, - view_tag: EncryptedAccountData::compute_view_tag( - &attacker_keys.npk(), - &attacker_keys.vpk(), - ), - ssk: attacker_ssk, + vpk: attacker_keys.vpk(), + random_seed: [0; 32], nsk: attacker_keys.nsk, membership_proof, identifier: 0, diff --git a/lez/common/src/block.rs b/lez/common/src/block.rs index b7c3e235..b22eec9f 100644 --- a/lez/common/src/block.rs +++ b/lez/common/src/block.rs @@ -55,6 +55,31 @@ pub struct Block { pub bedrock_status: BedrockStatus, } +impl Block { + /// Recomputes the hash from this block's contents, for integrity verification + /// against the value stored in `header.hash`. + #[must_use] + pub fn recompute_hash(&self) -> BlockHash { + HashableBlockData { + block_id: self.header.block_id, + prev_block_hash: self.header.prev_block_hash, + timestamp: self.header.timestamp, + transactions: self.body.transactions.clone(), + } + .compute_hash() + } + + /// Recomputes the signed hash from the block contents and checks the header + /// signature against `expected_pubkey`. Used to pin a peer zone's + /// block-signing key, so a block inscribed by anyone other than that zone's + /// sequencer is rejected even if it reached the channel. + #[must_use] + pub fn is_signed_by(&self, expected_pubkey: &lee::PublicKey) -> bool { + let hash = HashableBlockData::from(self.clone()).compute_hash(); + self.header.signature.is_valid_for(&hash.0, expected_pubkey) + } +} + impl Serialize for Block { fn serialize(&self, serializer: S) -> Result { crate::borsh_base64::serialize(self, serializer) @@ -76,10 +101,10 @@ pub struct HashableBlockData { } impl HashableBlockData { - /// The hash a block's signature is computed over. Signing and verifying both - /// go through this, so a peer's block-signing key can be re-checked. + /// Domain-separated hash of the block contents: `SHA256(PREFIX || borsh(self))`. + /// The single source of truth for both producing and verifying a block hash. #[must_use] - pub fn signing_hash(&self) -> BlockHash { + pub fn compute_hash(&self) -> BlockHash { const PREFIX: &[u8; 32] = b"/LEE/v0.3/Message/Block/\x00\x00\x00\x00\x00\x00\x00\x00"; let data_bytes = borsh::to_vec(self).unwrap(); @@ -91,13 +116,12 @@ impl HashableBlockData { ); bytes.extend_from_slice(PREFIX); bytes.extend_from_slice(&data_bytes); - OwnHasher::hash(&bytes) } #[must_use] pub fn into_pending_block(self, signing_key: &lee::PrivateKey) -> Block { - let hash = self.signing_hash(); + let hash = self.compute_hash(); let signature = lee::Signature::new(signing_key, &hash.0); Block { header: BlockHeader { @@ -126,18 +150,6 @@ impl From for HashableBlockData { } } -impl Block { - /// Recomputes the signed hash from the block contents and checks the header - /// signature against `expected_pubkey`. Used to pin a peer zone's - /// block-signing key, so a block inscribed by anyone other than that zone's - /// sequencer is rejected even if it reached the channel. - #[must_use] - pub fn is_signed_by(&self, expected_pubkey: &lee::PublicKey) -> bool { - let hash = HashableBlockData::from(self.clone()).signing_hash(); - self.header.signature.is_valid_for(&hash.0, expected_pubkey) - } -} - #[cfg(test)] mod tests { use crate::{HashType, block::HashableBlockData, test_utils}; @@ -151,4 +163,33 @@ mod tests { let block_from_bytes = borsh::from_slice::(&bytes).unwrap(); assert_eq!(hashable, block_from_bytes); } + + #[test] + fn recompute_hash_matches_header_for_well_formed_block() { + let key = lee::PrivateKey::try_new([7_u8; 32]).expect("valid key"); + let block = HashableBlockData { + block_id: 5, + prev_block_hash: HashType([9_u8; 32]), + timestamp: 42, + transactions: vec![test_utils::produce_dummy_empty_transaction()], + } + .into_pending_block(&key); + assert_eq!(block.recompute_hash(), block.header.hash); + } + + #[test] + fn recompute_hash_detects_tampering() { + let key = lee::PrivateKey::try_new([7_u8; 32]).expect("valid key"); + let block = HashableBlockData { + block_id: 5, + prev_block_hash: HashType([9_u8; 32]), + timestamp: 42, + transactions: vec![test_utils::produce_dummy_empty_transaction()], + } + .into_pending_block(&key); + + let mut tampered = block; + tampered.header.timestamp = 99; // header changed; stale hash no longer matches + assert_ne!(tampered.recompute_hash(), tampered.header.hash); + } } diff --git a/lez/common/src/transaction.rs b/lez/common/src/transaction.rs index b5aee648..b5f0bc87 100644 --- a/lez/common/src/transaction.rs +++ b/lez/common/src/transaction.rs @@ -92,9 +92,8 @@ impl LeeTransaction { Ok(diff) } - /// Computes the validated state diff without enforcing the system-account - /// restriction. Shared by [`Self::validate_on_state`] and - /// [`Self::execute_without_system_accounts_check_on_state`]. + /// Computes the validated state diff. Shared by [`Self::validate_on_state`] + /// (which adds the system-account guards) and [`Self::execute_on_state`]. fn compute_state_diff( &self, state: &V03State, @@ -129,16 +128,12 @@ impl LeeTransaction { Ok(self) } - /// Similar to [`Self::execute_check_on_state`], but skips the system-account guard. + /// Executes the transaction against the current state and applies the resulting diff, + /// without the system-account guards enforced by [`Self::execute_check_on_state`]. /// - /// FIXME: HOT FIX (testnet v0.2): the indexer replays blocks the sequencer already - /// accepted, including sequencer-generated deposit transactions that - /// legitimately modify the bridge account. The `TransactionOrigin::Sequencer` - /// tag that lets the sequencer bypass the guard is not carried in the block, - /// so the indexer cannot yet distinguish deposit txs from user txs. - /// - /// REMOVE ME when the indexer can authenticate deposit transactions. - pub fn execute_without_system_accounts_check_on_state( + /// The indexer replays blocks the sequencer already validated and inscribed on Bedrock, + /// so it trusts those inscriptions and re-derives state without re-validating them. + pub fn execute_on_state( self, state: &mut V03State, block_id: BlockId, diff --git a/lez/configs/docker-all-in-one/indexer_config.json b/lez/configs/docker-all-in-one/indexer_config.json index c1ff65b0..5791f64a 100644 --- a/lez/configs/docker-all-in-one/indexer_config.json +++ b/lez/configs/docker-all-in-one/indexer_config.json @@ -3,5 +3,6 @@ "bedrock_config": { "addr": "http://logos-blockchain-node-0:18080" }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101" + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "allow_chain_reset": true } diff --git a/lez/indexer/core/Cargo.toml b/lez/indexer/core/Cargo.toml index 84a85184..14941773 100644 --- a/lez/indexer/core/Cargo.toml +++ b/lez/indexer/core/Cargo.toml @@ -32,6 +32,7 @@ futures.workspace = true url.workspace = true logos-blockchain-core.workspace = true serde_json.workspace = true +thiserror.workspace = true async-stream.workspace = true tokio.workspace = true risc0-zkvm.workspace = true @@ -39,6 +40,4 @@ hex.workspace = true [dev-dependencies] tempfile.workspace = true -authenticated_transfer_core.workspace = true ping_core.workspace = true -bridge_lock_core.workspace = true diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index 5cefd256..8c974399 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -2,17 +2,40 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context as _, Result}; use common::{ - block::{BedrockStatus, Block}, + HashType, + block::{BedrockStatus, Block, BlockHeader}, transaction::{LeeTransaction, clock_invocation}, }; -use lee::{Account, AccountId, V03State}; +use lee::{Account, AccountId, GENESIS_BLOCK_ID, V03State}; use lee_core::BlockId; -use log::info; +use log::warn; use logos_blockchain_core::header::HeaderId; use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; use tokio::sync::RwLock; +use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; + +struct Tip { + block_id: u64, + hash: HashType, +} + +/// Outcome of feeding a parsed L2 block to the validated tip. +pub enum AcceptOutcome { + /// Chained and applied; tip and L1 read cursor both advance. + Applied, + /// A duplicate re-delivery of the current tip. Just L2 advances. + AlreadyApplied, + /// Did not chain or failed to apply; tip stays frozen, stall recorded. + Parked(BlockIngestError), + /// Chained but failed to apply, possibly transiently + /// ([`BlockIngestError::is_retryable`]); nothing recorded, tip and state + /// untouched. The caller retries and parks via + /// [`IndexerStore::record_stall`] once it gives up. + RetryableFailure(BlockIngestError), +} + #[derive(Clone)] pub struct IndexerStore { dbio: Arc, @@ -124,6 +147,36 @@ impl IndexerStore { Ok(()) } + /// The L1 inscription slot of the validated tip, written atomically with it + /// by [`Self::accept_block`]. `None` on a cold store or one written before + /// the slot was recorded. + pub fn get_tip_slot(&self) -> Result> { + Ok(self.dbio.get_meta_tip_slot_in_db()?.map(Slot::from)) + } + + pub fn get_stall_reason(&self) -> Result> { + let Some(bytes) = self.dbio.get_stall_reason_bytes()? else { + return Ok(None); + }; + let stall: Option = + serde_json::from_slice(&bytes).context("Failed to deserialize stored stall reason")?; + Ok(stall) + } + + pub fn set_stall_reason(&self, stall: &Option) -> Result<()> { + let bytes = serde_json::to_vec(stall).context("Failed to serialize stall reason")?; + self.dbio.put_stall_reason_bytes(&bytes)?; + Ok(()) + } + + /// Clears a recorded stall marker if one is present, skipping the write otherwise. + fn clear_stall_if_present(&self) -> Result<()> { + if self.get_stall_reason()?.is_some() { + self.set_stall_reason(&None)?; + } + Ok(()) + } + /// Recalculation of final state directly from DB. /// /// Used for indexer healthcheck. @@ -145,137 +198,251 @@ impl IndexerStore { .get_account_by_id(*account_id)) } - pub async fn put_block(&self, mut block: Block, l1_header: HeaderId) -> Result<()> { - info!("Applying block {}", block.header.block_id); - { - let mut state_guard = self.current_state.write().await; + /// The last successfully applied block, or `None` on a cold store. + /// Read fresh from the store each call. + fn validated_tip(&self) -> Result> { + let Some(block_id) = self.dbio.get_meta_last_block_id_in_db()? else { + return Ok(None); + }; + let Some(block) = self.dbio.get_block(block_id)? else { + return Ok(None); + }; + Ok(Some(Tip { + block_id, + hash: block.header.hash, + })) + } - let (clock_tx, user_txs) = block - .body - .transactions - .split_last() - .ok_or_else(|| anyhow::anyhow!("Block has no transactions"))?; - - anyhow::ensure!( - *clock_tx == LeeTransaction::Public(clock_invocation(block.header.timestamp)), - "Last transaction in block must be the clock invocation for the block timestamp" - ); - - let is_genesis = block.header.block_id == 1; - for transaction in user_txs { - if is_genesis { - let genesis_tx = match transaction { - LeeTransaction::Public(public_tx) => public_tx, - LeeTransaction::PrivacyPreserving(_) - | LeeTransaction::ProgramDeployment(_) => { - anyhow::bail!("Genesis block should contain only public transactions") - } - }; - state_guard - .transition_from_public_transaction( - genesis_tx, - block.header.block_id, - block.header.timestamp, - ) - .context("Failed to execute genesis public transaction")?; - } else { - transaction - .clone() - .transaction_stateless_check()? - // FIXME: HOT FIX (testnet v0.2): does not check for system account updates due to - // sequencer-generated deposit tx'es; - // CHANGE ME back to `execute_check_on_state` when the indexer can authenticate deposit transactions - .execute_without_system_accounts_check_on_state( - &mut state_guard, - block.header.block_id, - block.header.timestamp, - )?; - } + /// Record the stall reason. + /// + /// - First stall is stored verbatim + /// - Subsequent stalls only bump `orphans_since`, preserving the original cause. + pub fn record_stall( + &self, + header: Option<&BlockHeader>, + l1_slot: Slot, + error: BlockIngestError, + ) -> Result<()> { + let stall = match self.get_stall_reason()? { + Some(mut existing) => { + existing.orphans_since = existing.orphans_since.saturating_add(1); + existing } + None => StallReason { + // need to map out of `header` because they are not ser/de + block_id: header.map(|h| h.block_id), + block_hash: header.map(|h| h.hash), + prev_block_hash: header.map(|h| h.prev_block_hash), + first_seen: header.map(|h| h.timestamp), + l1_slot, + error, + orphans_since: 0, + }, + }; + self.set_stall_reason(&Some(stall)) + } - // Apply the clock invocation directly (it is expected to modify clock accounts). - let LeeTransaction::Public(clock_public_tx) = clock_tx else { - anyhow::bail!("Clock invocation must be a public transaction"); - }; - state_guard.transition_from_public_transaction( - clock_public_tx, - block.header.block_id, - block.header.timestamp, - )?; + /// Validates `block` against the tip and, if it chains, applies it atomically + /// (scratch clone, commit only on full success) and advances the tip. + /// Retryable apply failures return `RetryableFailure` without recording a stall + /// or touching state; other failures record the stall and return `Parked`. + pub async fn accept_block(&self, block: &Block, l1_slot: Slot) -> Result { + let tip = self.validated_tip()?; + + // Re-delivery of an already-applied block is idempotent, not a divergence + if let Some(tip) = &tip + && block.header.block_id <= tip.block_id + && let Some(stored) = self.get_block_at_id(block.header.block_id)? + && stored.header.hash == block.header.hash + { + return Ok(AcceptOutcome::AlreadyApplied); } - // ToDo: Currently we are fetching only finalized blocks - // if it changes, the following lines need to be updated - // to represent correct block finality - block.bedrock_status = BedrockStatus::Finalized; + if let Err(err) = validate_against_tip(tip.as_ref(), block) { + self.record_stall(Some(&block.header), l1_slot, err.clone())?; + return Ok(AcceptOutcome::Parked(err)); + } - info!("Putting block {} into DB", block.header.block_id); - Ok(self.dbio.put_block(&block, l1_header.into())?) + // TODO: we use scratch state to be atomic, but need to revisit how expensive a clone is + let mut scratch = self.current_state.read().await.clone(); + if let Err(err) = apply_block_to_scratch(block, &mut scratch) { + if err.is_retryable() { + return Ok(AcceptOutcome::RetryableFailure(err)); + } + self.record_stall(Some(&block.header), l1_slot, err.clone())?; + return Ok(AcceptOutcome::Parked(err)); + } + + let mut stored = block.clone(); + stored.bedrock_status = BedrockStatus::Finalized; + self.dbio + .put_block(&stored, [0_u8; 32], l1_slot.into_inner(), &scratch) + .context("Failed to persist accepted block")?; + + // Commit in-memory state (infallible) only after the DB write succeeded. + *self.current_state.write().await = scratch; + // Best-effort: the block is durably applied, so a failed stall clear must not + // fail the apply. It self-heals on the next clear. + if let Err(err) = self.clear_stall_if_present() { + warn!("Failed to clear stall marker after applying block: {err:#}"); + } + Ok(AcceptOutcome::Applied) + } +} + +/// Checks that `block` is the valid continuation of `tip`: hash integrity, +/// then block-id continuity, then `prev_block_hash` linkage. A `None` tip +/// (cold store) expects the genesis block. +fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIngestError> { + let computed = block.recompute_hash(); + if computed != block.header.hash { + return Err(BlockIngestError::HashMismatch { + computed, + header: block.header.hash, + }); + } + + match tip { + None => { + if block.header.block_id != GENESIS_BLOCK_ID { + return Err(BlockIngestError::UnexpectedBlockId { + expected: GENESIS_BLOCK_ID, + got: block.header.block_id, + }); + } + } + Some(tip) => { + let expected = tip + .block_id + .checked_add(1) + .expect("block id should not overflow"); + if block.header.block_id != expected { + return Err(BlockIngestError::UnexpectedBlockId { + expected, + got: block.header.block_id, + }); + } + if block.header.prev_block_hash != tip.hash { + return Err(BlockIngestError::BrokenChainLink { + expected_prev: tip.hash, + got_prev: block.header.prev_block_hash, + }); + } + } + } + Ok(()) +} + +/// Applies a block's transactions to `state`, mapping every failure to a +/// [`BlockIngestError`] so the caller can park rather than crash. Operates on a +/// scratch state; the caller commits only on `Ok`. +fn apply_block_to_scratch(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { + let (clock_tx, user_txs) = block + .body + .transactions + .split_last() + .ok_or(BlockIngestError::EmptyBlock)?; + + let expected_clock = LeeTransaction::Public(clock_invocation(block.header.timestamp)); + if *clock_tx != expected_clock { + return Err(BlockIngestError::InvalidClockTransaction); + } + + let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; + for (tx_index, transaction) in user_txs.iter().enumerate() { + let state_transition = |err: anyhow::Error| BlockIngestError::StateTransition { + tx_index: tx_index.try_into().expect("tx index fits in u64"), + reason: format!("{err:#}"), + }; + if is_genesis { + let LeeTransaction::Public(public_tx) = transaction else { + return Err(BlockIngestError::NonPublicGenesisTransaction); + }; + state + .transition_from_public_transaction( + public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| state_transition(err.into()))?; + } else { + transaction + .clone() + .execute_on_state(state, block.header.block_id, block.header.timestamp) + .map_err(|err| state_transition(err.into()))?; + } + } + + let LeeTransaction::Public(clock_public_tx) = clock_tx else { + return Err(BlockIngestError::InvalidClockTransaction); + }; + state + .transition_from_public_transaction( + clock_public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| BlockIngestError::StateTransition { + tx_index: user_txs.len().try_into().expect("tx index fits in u64"), + reason: format!("{:#}", anyhow::Error::from(err)), + })?; + + Ok(()) +} + +#[cfg(test)] +mod stall_reason_tests { + use common::HashType; + + use super::*; + use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; + + #[tokio::test] + async fn stall_reason_roundtrips_and_clears() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + assert!(store.get_stall_reason().expect("get").is_none()); + + let stall = StallReason { + block_id: Some(7), + block_hash: Some(HashType([1_u8; 32])), + prev_block_hash: Some(HashType([2_u8; 32])), + l1_slot: Slot::from(42), + error: BlockIngestError::StateTransition { + tx_index: 0, + reason: "boom".to_owned(), + }, + first_seen: Some(99), + orphans_since: 3, + }; + store.set_stall_reason(&Some(stall)).expect("set stall"); + + let got = store.get_stall_reason().expect("get").expect("present"); + assert_eq!(got.block_id, Some(7)); + assert_eq!(got.orphans_since, 3); + assert!(matches!( + got.error, + BlockIngestError::StateTransition { .. } + )); + assert_eq!(got.block_hash, Some(HashType([1_u8; 32]))); + assert_eq!(got.prev_block_hash, Some(HashType([2_u8; 32]))); + assert_eq!(got.l1_slot, Slot::from(42)); + assert_eq!(got.first_seen, Some(99)); + + store.set_stall_reason(&None).expect("clear"); + assert!(store.get_stall_reason().expect("get").is_none()); } } #[cfg(test)] mod tests { - use common::{HashType, block::HashableBlockData}; + use common::test_utils::{create_transaction_native_token_transfer, produce_dummy_block}; use tempfile::tempdir; use testnet_initial_state::initial_pub_accounts_private_keys; use super::*; - struct TestFixture { - storage: IndexerStore, - from: AccountId, - to: AccountId, - _home: tempfile::TempDir, - } - - #[expect( - clippy::arithmetic_side_effects, - reason = "test helper with bounded inputs" - )] - async fn store_with_transfer_blocks( - block_count: u64, - prev_hash: Option, - ) -> TestFixture { - let home = tempdir().unwrap(); - let storage = IndexerStore::open_db(home.path(), Vec::new()).unwrap(); - - let initial_accounts = initial_pub_accounts_private_keys(); - let from = initial_accounts[0].account_id; - let to = initial_accounts[1].account_id; - let sign_key = initial_accounts[0].pub_sign_key.clone(); - - let mut prev_hash = prev_hash; - for i in 0..block_count { - let tx = common::test_utils::create_transaction_native_token_transfer( - from, - u128::from(i), - to, - 10, - &sign_key, - ); - let block_id = i + 1; - - let next_block = common::test_utils::produce_dummy_block(block_id, prev_hash, vec![tx]); - prev_hash = Some(next_block.header.hash); - - storage - .put_block( - next_block, - HeaderId::from([u8::try_from(i + 1).unwrap(); 32]), - ) - .await - .unwrap(); - } - - TestFixture { - storage, - from, - to, - _home: home, - } - } - #[test] fn correct_startup() { let home = tempdir().unwrap(); @@ -288,75 +455,474 @@ mod tests { } #[tokio::test] - async fn state_transition() { + async fn accept_block_applies_transfers_and_advances_tip() { let home = tempdir().unwrap(); - let storage = IndexerStore::open_db(home.as_ref(), Vec::new()).unwrap(); + let store = IndexerStore::open_db(home.as_ref(), Vec::new()).unwrap(); let initial_accounts = initial_pub_accounts_private_keys(); let from = initial_accounts[0].account_id; let to = initial_accounts[1].account_id; let sign_key = initial_accounts[0].pub_sign_key.clone(); - let clock_tx = LeeTransaction::Public(clock_invocation(0)); - let genesis_block_data = HashableBlockData { - block_id: 1, - prev_block_hash: HashType::default(), - timestamp: 0, - transactions: vec![clock_tx], - }; - let genesis_block = genesis_block_data - .into_pending_block(&common::test_utils::sequencer_sign_key_for_testing()); - let mut prev_hash = Some(genesis_block.header.hash); - storage - .put_block(genesis_block, HeaderId::from([0_u8; 32])) - .await - .unwrap(); + // Genesis (block 1): clock-only. + let genesis = produce_dummy_block(1, None, vec![]); + let mut prev_hash = genesis.header.hash; + assert!(matches!( + store.accept_block(&genesis, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); - for i in 0..10_u128 { - let tx = common::test_utils::create_transaction_native_token_transfer( - from, i, to, 10, &sign_key, - ); - let block_id = u64::try_from(i + 1).unwrap(); - let next_block = common::test_utils::produce_dummy_block(block_id, prev_hash, vec![tx]); - prev_hash = Some(next_block.header.hash); - storage - .put_block( - next_block, - HeaderId::from([u8::try_from(i + 1).unwrap(); 32]), - ) - .await - .unwrap(); + // Blocks 2..=11: one native transfer of 10 each (nonces 0..=9). + for i in 0..10_u64 { + let tx = create_transaction_native_token_transfer(from, i.into(), to, 10, &sign_key); + let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); + prev_hash = block.header.hash; + assert!(matches!( + store.accept_block(&block, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); } - let acc1_val = storage.account_current_state(&from).await.unwrap(); - let acc2_val = storage.account_current_state(&to).await.unwrap(); - - assert_eq!(acc1_val.balance, 9900); - assert_eq!(acc2_val.balance, 20100); + assert_eq!( + store.account_current_state(&from).await.unwrap().balance, + 9900 + ); + assert_eq!( + store.account_current_state(&to).await.unwrap().balance, + 20100 + ); + // Tip advanced to the last applied block; a clean run leaves no stall. + assert_eq!(store.get_last_block_id().unwrap(), Some(11)); + assert!(store.get_stall_reason().unwrap().is_none()); } #[tokio::test] - async fn account_state_at_block() { - let TestFixture { - storage, - from, - to, - _home, - } = store_with_transfer_blocks(10, None).await; + async fn account_state_at_block_reflects_history() { + let home = tempdir().unwrap(); + let store = IndexerStore::open_db(home.as_ref(), Vec::new()).unwrap(); - let acc1_at_1 = storage.account_state_at_block(&from, 1).unwrap(); - let acc2_at_1 = storage.account_state_at_block(&to, 1).unwrap(); - assert_eq!(acc1_at_1.balance, 9990); - assert_eq!(acc2_at_1.balance, 20010); + let initial_accounts = initial_pub_accounts_private_keys(); + let from = initial_accounts[0].account_id; + let to = initial_accounts[1].account_id; + let sign_key = initial_accounts[0].pub_sign_key.clone(); - let acc1_at_5 = storage.account_state_at_block(&from, 5).unwrap(); - let acc2_at_5 = storage.account_state_at_block(&to, 5).unwrap(); - assert_eq!(acc1_at_5.balance, 9950); - assert_eq!(acc2_at_5.balance, 20050); + let genesis = produce_dummy_block(1, None, vec![]); + let mut prev_hash = genesis.header.hash; + store.accept_block(&genesis, Slot::from(0)).await.unwrap(); - let acc1_at_9 = storage.account_state_at_block(&from, 9).unwrap(); - let acc2_at_9 = storage.account_state_at_block(&to, 9).unwrap(); - assert_eq!(acc1_at_9.balance, 9910); - assert_eq!(acc2_at_9.balance, 20090); + for i in 0..10_u64 { + let tx = create_transaction_native_token_transfer(from, i.into(), to, 10, &sign_key); + let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); + prev_hash = block.header.hash; + store.accept_block(&block, Slot::from(0)).await.unwrap(); + } + + // State at block N is inclusive of block N. + // Block 1 (genesis, clock-only): no transfers yet. + assert_eq!( + store.account_state_at_block(&from, 1).unwrap().balance, + 10000 + ); + assert_eq!(store.account_state_at_block(&to, 1).unwrap().balance, 20000); + // Through block 5: 4 transfers applied (blocks 2..=5). + assert_eq!( + store.account_state_at_block(&from, 5).unwrap().balance, + 9960 + ); + assert_eq!(store.account_state_at_block(&to, 5).unwrap().balance, 20040); + // Through block 9: 8 transfers applied (blocks 2..=9). + assert_eq!( + store.account_state_at_block(&from, 9).unwrap().balance, + 9920 + ); + assert_eq!(store.account_state_at_block(&to, 9).unwrap().balance, 20080); + } +} + +#[cfg(test)] +mod accept_tests { + use common::{HashType, block::HashableBlockData, test_utils::produce_dummy_block}; + + use super::*; + use crate::ingest_error::BlockIngestError; + + fn signing_key() -> lee::PrivateKey { + lee::PrivateKey::try_new([7_u8; 32]).expect("valid key") + } + + // A block with a correct hash but empty body โ€” enough to exercise the + // acceptance checks (id/link/hash), which run before any state application. + fn valid_hash_block(block_id: u64, prev: HashType) -> common::block::Block { + HashableBlockData { + block_id, + prev_block_hash: prev, + timestamp: 0, + transactions: vec![], + } + .into_pending_block(&signing_key()) + } + + #[tokio::test] + async fn non_genesis_first_block_parks_with_unexpected_id() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + let block = valid_hash_block(2, HashType([0_u8; 32])); + let outcome = store + .accept_block(&block, Slot::from(0)) + .await + .expect("accept"); + + assert!(matches!( + outcome, + AcceptOutcome::Parked(BlockIngestError::UnexpectedBlockId { + expected: 1, + got: 2 + }) + )); + let stall = store.get_stall_reason().expect("get").expect("present"); + assert_eq!(stall.block_id, Some(2)); + assert_eq!(stall.orphans_since, 0); + } + + #[tokio::test] + async fn hash_mismatch_parks() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + let mut block = valid_hash_block(1, HashType([0_u8; 32])); + block.header.timestamp = 999; // invalidates the stored hash + + let outcome = store + .accept_block(&block, Slot::from(0)) + .await + .expect("accept"); + assert!(matches!( + outcome, + AcceptOutcome::Parked(BlockIngestError::HashMismatch { .. }) + )); + } + + #[tokio::test] + async fn second_break_bumps_orphan_count_and_keeps_first() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + let first = valid_hash_block(2, HashType([0_u8; 32])); + store + .accept_block(&first, Slot::from(0)) + .await + .expect("accept"); + let second = valid_hash_block(3, HashType([0_u8; 32])); + store + .accept_block(&second, Slot::from(0)) + .await + .expect("accept"); + + let stall = store.get_stall_reason().expect("get").expect("present"); + assert_eq!(stall.block_id, Some(2), "first stall preserved"); + assert_eq!(stall.orphans_since, 1, "second break counted as orphan"); + } + + #[tokio::test] + async fn deserialize_break_records_stall_without_header() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + store + .record_stall( + None, + Slot::from(0), + BlockIngestError::Deserialize("bad bytes".to_owned()), + ) + .expect("record"); + + let stall = store.get_stall_reason().expect("get").expect("present"); + assert_eq!(stall.block_id, None); + assert!(matches!(stall.error, BlockIngestError::Deserialize(_))); + } + + #[tokio::test] + async fn parks_then_recovers_on_valid_continuation() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + // Genesis (block 1, clock-only) applies and advances the tip. + let genesis = produce_dummy_block(1, None, vec![]); + assert!(matches!( + store.accept_block(&genesis, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + + // A block that skips ahead (id 3 while the tip is 1) parks the indexer. + let bad = produce_dummy_block(3, Some(genesis.header.hash), vec![]); + assert!(matches!( + store.accept_block(&bad, Slot::from(0)).await.unwrap(), + AcceptOutcome::Parked(BlockIngestError::UnexpectedBlockId { + expected: 2, + got: 3 + }) + )); + assert!( + store.get_stall_reason().unwrap().is_some(), + "indexer should be parked after the bad block" + ); + assert_eq!( + store.get_last_block_id().unwrap(), + Some(1), + "validated tip must stay frozen at genesis while parked" + ); + + // The valid continuation (block 2 chaining on genesis) recovers the chain. + let next = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + assert!(matches!( + store.accept_block(&next, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + assert!( + store.get_stall_reason().unwrap().is_none(), + "stall reason must clear on recovery" + ); + assert_eq!( + store.get_last_block_id().unwrap(), + Some(2), + "tip must advance to the recovered block" + ); + } + + #[tokio::test] + async fn accept_block_records_tip_inscription_slot() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + assert_eq!(store.get_tip_slot().expect("get"), None); + + let genesis = produce_dummy_block(1, None, vec![]); + store + .accept_block(&genesis, Slot::from(1_000)) + .await + .expect("accept"); + assert_eq!(store.get_tip_slot().expect("get"), Some(Slot::from(1_000))); + + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![]); + store + .accept_block(&block2, Slot::from(1_005)) + .await + .expect("accept"); + assert_eq!(store.get_tip_slot().expect("get"), Some(Slot::from(1_005))); + + // A parked block freezes the tip, so its slot must not advance either. + let bad = produce_dummy_block(4, Some(block2.header.hash), vec![]); + assert!(matches!( + store.accept_block(&bad, Slot::from(1_010)).await.unwrap(), + AcceptOutcome::Parked(_) + )); + assert_eq!(store.get_tip_slot().expect("get"), Some(Slot::from(1_005))); + + // Neither must a re-delivered old block move it. + assert!(matches!( + store + .accept_block(&genesis, Slot::from(1_015)) + .await + .unwrap(), + AcceptOutcome::AlreadyApplied + )); + assert_eq!(store.get_tip_slot().expect("get"), Some(Slot::from(1_005))); + } + + #[tokio::test] + async fn redelivered_tip_block_is_idempotent_not_parked() { + use testnet_initial_state::initial_pub_accounts_private_keys; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let genesis = produce_dummy_block(1, None, vec![]); + store + .accept_block(&genesis, Slot::from(0)) + .await + .expect("accept genesis"); + + // Block 2: a single transfer of 10. + let tx = common::test_utils::create_transaction_native_token_transfer( + from, 0, to, 10, &sign_key, + ); + let block = produce_dummy_block(2, Some(genesis.header.hash), vec![tx]); + assert!(matches!( + store.accept_block(&block, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + let balance_after = store.account_current_state(&from).await.unwrap().balance; + + // Re-deliver the exact same block: idempotent skip, no state change, no park. + assert!(matches!( + store.accept_block(&block, Slot::from(0)).await.unwrap(), + AcceptOutcome::AlreadyApplied + )); + assert_eq!( + store.account_current_state(&from).await.unwrap().balance, + balance_after, + "re-delivered block must not be applied twice" + ); + assert_eq!( + store.get_last_block_id().unwrap(), + Some(2), + "tip must stay at the already-applied block" + ); + assert!( + store.get_stall_reason().unwrap().is_none(), + "a benign duplicate must not park the indexer" + ); + } + + #[tokio::test] + async fn redelivered_block_below_tip_is_idempotent_not_parked() { + use testnet_initial_state::initial_pub_accounts_private_keys; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + // Build a short chain: genesis (1) -> block 2 -> block 3, so the tip is 3. + let genesis = produce_dummy_block(1, None, vec![]); + store + .accept_block(&genesis, Slot::from(0)) + .await + .expect("accept genesis"); + + let tx2 = common::test_utils::create_transaction_native_token_transfer( + from, 0, to, 10, &sign_key, + ); + let block2 = produce_dummy_block(2, Some(genesis.header.hash), vec![tx2]); + assert!(matches!( + store.accept_block(&block2, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + + let tx3 = common::test_utils::create_transaction_native_token_transfer( + from, 1, to, 10, &sign_key, + ); + let block3 = produce_dummy_block(3, Some(block2.header.hash), vec![tx3]); + assert!(matches!( + store.accept_block(&block3, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + + let balance_after = store.account_current_state(&from).await.unwrap().balance; + + // Re-deliver block 2 (id below the tip): a re-delivery, not a divergence. + assert!(matches!( + store.accept_block(&block2, Slot::from(0)).await.unwrap(), + AcceptOutcome::AlreadyApplied + )); + assert_eq!( + store.account_current_state(&from).await.unwrap().balance, + balance_after, + "re-delivered block below the tip must not be applied again" + ); + assert_eq!( + store.get_last_block_id().unwrap(), + Some(3), + "tip must stay at the current head" + ); + assert!( + store.get_stall_reason().unwrap().is_none(), + "a benign re-delivery must not park the indexer" + ); + } + + #[tokio::test] + async fn accept_block_snapshots_state_at_breakpoint_interval() { + use testnet_initial_state::initial_pub_accounts_private_keys; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let genesis = produce_dummy_block(1, None, vec![]); + assert!(matches!( + store.accept_block(&genesis, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + let mut prev_hash = genesis.header.hash; + + // Blocks 2..=101: one transfer of 1 each; block 100 crosses the interval. + for i in 0..100_u64 { + let tx = common::test_utils::create_transaction_native_token_transfer( + from, + i.into(), + to, + 1, + &sign_key, + ); + let block = produce_dummy_block(i + 2, Some(prev_hash), vec![tx]); + prev_hash = block.header.hash; + assert!(matches!( + store.accept_block(&block, Slot::from(0)).await.unwrap(), + AcceptOutcome::Applied + )); + } + + // Snapshot at block 100 = genesis + 99 transfers, written with the block. + let bp1 = store.dbio.get_breakpoint(1).expect("breakpoint 1 present"); + assert_eq!(bp1.get_account_by_id(from).balance, 10000 - 99); + + // The #605 restart: reopening past the boundary must work. + drop(store); + let reopened = IndexerStore::open_db(dir.path(), Vec::new()).expect("reopen"); + assert_eq!(reopened.last_block().unwrap(), Some(101)); + } + + #[tokio::test] + async fn transient_apply_failure_returns_retryable_failure_without_stall() { + use testnet_initial_state::initial_pub_accounts_private_keys; + + let dir = tempfile::tempdir().expect("tempdir"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); + + let accounts = initial_pub_accounts_private_keys(); + let from = accounts[0].account_id; + let to = accounts[1].account_id; + let sign_key = accounts[0].pub_sign_key.clone(); + + let genesis = produce_dummy_block(1, None, vec![]); + store + .accept_block(&genesis, Slot::from(0)) + .await + .expect("accept genesis"); + + // Overdraft: rejected during execution โ†’ StateTransition โ†’ retryable. + let tx = common::test_utils::create_transaction_native_token_transfer( + from, + 0, + to, + 1_000_000_000, + &sign_key, + ); + let block = produce_dummy_block(2, Some(genesis.header.hash), vec![tx]); + let outcome = store.accept_block(&block, Slot::from(0)).await.unwrap(); + + assert!(matches!( + outcome, + AcceptOutcome::RetryableFailure(BlockIngestError::StateTransition { .. }) + )); + assert!( + store.get_stall_reason().unwrap().is_none(), + "retryable failure must not persist a stall" + ); + assert_eq!(store.get_last_block_id().unwrap(), Some(1), "tip frozen"); } } diff --git a/lez/indexer/core/src/chain_consistency.rs b/lez/indexer/core/src/chain_consistency.rs new file mode 100644 index 00000000..df4b5ba6 --- /dev/null +++ b/lez/indexer/core/src/chain_consistency.rs @@ -0,0 +1,550 @@ +//! Startup check that the local store still belongs to the chain the +//! connected channel serves. + +use anyhow::Result; +use common::{HashType, block::Block}; +use futures::StreamExt as _; +use log::warn; +use logos_blockchain_zone_sdk::{Slot, ZoneMessage, adapter::Node as _}; + +use crate::IndexerCore; + +/// Upper bound on the channel reads of the startup consistency check. +const CHANNEL_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); + +/// Result of comparing the indexer's stored chain against the channel. +pub enum ChainConsistency { + /// Channel still serves our anchor block (the stored tip position, or the + /// parked block while stalled). + Consistent, + /// We could not determine the outcome due to one of: + /// + /// - cold store (no anchor to compare at) + /// - the channel served only blocks newer than the anchor + /// - or the channel read was inconclusive (timeout / error / empty stream) + /// + /// NOTE: None of these prove a reset, so the caller proceeds. + /// A genuine divergence is still caught later when the ingest loop tries to apply and parks. + Inconclusive, + /// Positive evidence that the channel is a different chain than the store. + /// + /// Details in [`ChainMismatch`], and impl's Display trait. + Inconsistent(ChainMismatch), +} + +/// The evidence behind a [`ChainConsistency::Inconsistent`]. +pub enum ChainMismatch { + /// The channel serves a different block at the anchor's id. + Block { + ours: (u64, HashType), + channel: (u64, HashType), + }, + /// The channel serves a block at/below the anchor's id past the anchor + /// slot; on the same chain those ids live at earlier slots. + ReinscribedBlock { + channel: (u64, HashType), + slot: Slot, + anchor_slot: Slot, + }, + /// The channel has content past the anchor slot but no longer the + /// inscription we anchored on. + AnchorSlotChanged { anchor_slot: Slot }, + /// The channel does not exist on the connected chain. + ChannelMissing, + /// The channel's history ends before the anchor slot. + ChannelBehindAnchor { tip_slot: Slot, anchor_slot: Slot }, +} + +impl std::fmt::Display for ChainMismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Block { ours, channel } => write!( + f, + "stored block {} {} != channel block {} {}", + ours.0, ours.1, channel.0, channel.1 + ), + Self::ReinscribedBlock { + channel, + slot, + anchor_slot, + } => write!( + f, + "channel re-serves block {} {} at slot {} past our anchor slot {}", + channel.0, + channel.1, + slot.into_inner(), + anchor_slot.into_inner() + ), + Self::AnchorSlotChanged { anchor_slot } => write!( + f, + "channel content at slot {} no longer includes the inscription we parked on", + anchor_slot.into_inner() + ), + Self::ChannelMissing => write!(f, "channel does not exist on the connected chain"), + Self::ChannelBehindAnchor { + tip_slot, + anchor_slot, + } => write!( + f, + "channel tip slot {} is behind our anchor slot {}", + tip_slot.into_inner(), + anchor_slot.into_inner() + ), + } + } +} + +/// A block that must still be inscribed at `slot` if the channel is the chain +/// the store was built from: the tip at the read cursor, or the recorded +/// parked block while stalled. +struct Anchor { + slot: Slot, + /// The anchor block's `(id, hash)`. + /// + /// `None` when parked on an undeserializable inscription (no header was recorded). + block: Option<(u64, HashType)>, +} + +impl Anchor { + /// Probes a channel message read at/after the anchor slot. + /// See [`IndexerCore::verify_chain_at_anchor`]. + pub fn probe_anchor_slot(&self, msg: &ZoneMessage, slot: Slot) -> AnchorProbe { + if slot < self.slot { + return AnchorProbe::KeepLooking; + } + let Some((anchor_id, anchor_hash)) = self.block else { + // Anchored on an undeserializable inscription: any message still + // present at that slot means the history is intact. + return if slot == self.slot { + AnchorProbe::SameChain + } else { + AnchorProbe::Mismatch(ChainMismatch::AnchorSlotChanged { + anchor_slot: self.slot, + }) + }; + }; + let ZoneMessage::Block(zone_block) = msg else { + return AnchorProbe::KeepLooking; + }; + let Ok(block) = borsh::from_slice::(&zone_block.data) else { + return AnchorProbe::KeepLooking; + }; + let (id, hash) = (block.header.block_id, block.header.hash); + if id == anchor_id { + return if hash == anchor_hash { + AnchorProbe::SameChain + } else { + AnchorProbe::Mismatch(ChainMismatch::Block { + ours: (anchor_id, anchor_hash), + channel: (id, hash), + }) + }; + } + if id > anchor_id { + return AnchorProbe::Bail; + } + if slot == self.slot { + // Older ids can share the anchor's slot on the same chain. + return AnchorProbe::KeepLooking; + } + // An id below the anchor served past the anchor slot is impossible on the + // same chain, even if the content is identical (deterministic genesis). + AnchorProbe::Mismatch(ChainMismatch::ReinscribedBlock { + channel: (id, hash), + slot, + anchor_slot: self.slot, + }) + } +} + +/// What a single channel message tells the anchored consistency check. +enum AnchorProbe { + /// The anchor is still in place: same chain. + SameChain, + Mismatch(ChainMismatch), + /// Only newer ids past the anchor: plausible on the same chain, so stop + /// scanning without a verdict. + Bail, + KeepLooking, +} + +#[expect( + clippy::multiple_inherent_impl, + reason = "split for clarity & isolation of relevant code" +)] +impl IndexerCore { + /// Verifies whether the channel still serves the same chain the store was built from. + /// This may change frequently during development where we reset the chain from time to + /// time in devnet/testnet, but we do not expect [`ChainConsistency::Inconsistent`] in + /// production. + /// + /// To compare the chains, we use an [`Anchor`] block that is either the parked L2 block + /// while stalled, or the tip L2 block at its own inscription L1 slot. + pub(crate) async fn verify_chain_consistency(&self) -> Result { + let Some(anchor) = self.get_startup_anchor()? else { + // empty or cold store: nothing to compare + return Ok(ChainConsistency::Inconclusive); + }; + + self.verify_chain_at_anchor(&anchor).await + } + + /// Builds the anchor for the startup check. + /// + /// - If stalled, returns the recorded _parked_ block + /// - If not stalled, returns the validated tip at its _own_ inscription slot. + /// - If the store is empty, returns `None`. + fn get_startup_anchor(&self) -> Result> { + if let Some(stall) = self.store.get_stall_reason()? { + return Ok(Some(Anchor { + slot: stall.l1_slot, + block: stall.block_id.zip(stall.block_hash), + })); + } + + // not stalled, so anchor on the tip at its own inscription slot + let Some(slot) = self.store.get_tip_slot()?.or(self.store.get_zone_cursor()?) else { + return Ok(None); + }; + let Some(tip_id) = self.store.get_last_block_id()? else { + return Ok(None); + }; + let Some(tip) = self.store.get_block_at_id(tip_id)? else { + return Ok(None); + }; + Ok(Some(Anchor { + slot, + block: Some((tip_id, tip.header.hash)), + })) + } + + /// Verifies the channel still carries the anchor block at its slot. + /// + /// The anchor was finalized at `anchor.slot`, so the same chain must still + /// serve it there, while a reset chain re-inscribes its content only at + /// later wall-clock slots. + /// + /// Only positive evidence of a different chain yields `Inconsistent`. + /// Absence of data stays `Inconclusive`. + async fn verify_chain_at_anchor(&self, anchor: &Anchor) -> Result { + match self.node.channel_state(self.config.channel_id).await { + Ok(state) => { + if let Some(mismatch) = frontier_verdict(anchor.slot, state.map(|s| s.tip_slot)) { + return Ok(ChainConsistency::Inconsistent(mismatch)); + } + } + Err(err) => { + warn!("Failed to read channel state for the consistency check: {err:#}"); + } + } + + // `next_messages` is exclusive, so `slot - 1` includes the anchor slot. + let Some(from_slot) = anchor.slot.into_inner().checked_sub(1) else { + return Ok(ChainConsistency::Inconclusive); + }; + + let scan = async { + let stream = self + .zone_indexer + .next_messages(Some(Slot::from(from_slot))) + .await?; + let mut stream = std::pin::pin!(stream); + + while let Some((msg, slot)) = stream.next().await { + match anchor.probe_anchor_slot(&msg, slot) { + AnchorProbe::SameChain => return Ok(Some(ChainConsistency::Consistent)), + AnchorProbe::Mismatch(mismatch) => { + return Ok(Some(ChainConsistency::Inconsistent(mismatch))); + } + AnchorProbe::Bail => return Ok(Some(ChainConsistency::Inconclusive)), + AnchorProbe::KeepLooking => { /* dont do anything */ } + } + } + Ok::<_, anyhow::Error>(None) + }; + + match tokio::time::timeout(CHANNEL_READ_TIMEOUT, scan).await { + Ok(Ok(Some(outcome))) => Ok(outcome), + Ok(Ok(None)) => Ok(ChainConsistency::Inconclusive), + Ok(Err(err)) => { + warn!( + "Failed to read the anchor slot for the consistency check; proceeding: {err:#}" + ); + Ok(ChainConsistency::Inconclusive) + } + Err(_elapsed) => { + warn!("Timed out reading the anchor slot for the consistency check; proceeding"); + Ok(ChainConsistency::Inconclusive) + } + } + } +} + +/// Checks the channel frontier against the anchor slot. +/// +/// The anchor block was finalized at `anchor_slot`, so on the same chain the +/// channel tip can never be behind it, and the channel must exist. +fn frontier_verdict(anchor_slot: Slot, channel_tip_slot: Option) -> Option { + match channel_tip_slot { + None => Some(ChainMismatch::ChannelMissing), + Some(tip_slot) if tip_slot < anchor_slot => Some(ChainMismatch::ChannelBehindAnchor { + tip_slot, + anchor_slot, + }), + Some(_) => None, + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use common::block::HashableBlockData; + use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription}; + use logos_blockchain_zone_sdk::ZoneBlock; + + use super::*; + use crate::{ + BlockIngestError, + block_store::AcceptOutcome, + config::{ChannelId, ClientConfig, IndexerConfig}, + }; + + fn unreachable_core(dir: &std::path::Path) -> IndexerCore { + let config = IndexerConfig { + consensus_info_polling_interval: Duration::from_secs(1), + bedrock_config: ClientConfig { + addr: "http://localhost:1".parse().expect("url"), + auth: None, + }, + channel_id: ChannelId::from([1; 32]), + allow_chain_reset: false, + cross_zone: None, + bridge_lock_holdings: Vec::new(), + }; + IndexerCore::open(config, dir).expect("open core") + } + + fn test_block(block_id: u64, timestamp: u64) -> Block { + HashableBlockData { + block_id, + prev_block_hash: HashType([0; 32]), + timestamp, + transactions: vec![], + } + .into_pending_block(&lee::PrivateKey::try_new([7; 32]).expect("valid key")) + } + + fn block_msg(block: &Block) -> ZoneMessage { + let bytes = borsh::to_vec(block).expect("serialize"); + ZoneMessage::Block(ZoneBlock { + id: MsgId::from([0_u8; 32]), + data: Inscription::try_from(bytes.as_slice()).expect("inscription"), + }) + } + + fn anchor_for(block: &Block, slot: Slot) -> Anchor { + Anchor { + slot, + block: Some((block.header.block_id, block.header.hash)), + } + } + + #[tokio::test] + async fn cold_store_is_inconclusive() { + // An empty store has no cursor, so there is nothing to compare: the check + // must be Inconclusive (not Consistent), and it returns before any L1 read. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[tokio::test] + async fn parked_store_with_unreachable_node_is_inconclusive() { + // Network failure is not evidence of a reset: a parked store must stay + // parked (Inconclusive), not error out or trip the wipe path. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + let parked = test_block(5, 42); + core.store + .record_stall( + Some(&parked.header), + Slot::from(1_000), + BlockIngestError::EmptyBlock, + ) + .expect("record stall"); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[tokio::test] + async fn caught_up_store_with_unreachable_node_is_inconclusive() { + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + let genesis = common::test_utils::produce_dummy_block(1, None, vec![]); + assert!(matches!( + core.store + .accept_block(&genesis, Slot::from(1_000)) + .await + .expect("accept"), + AcceptOutcome::Applied + )); + core.store + .set_zone_cursor(&Slot::from(1_000)) + .expect("set cursor"); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[tokio::test] + async fn startup_anchor_prefers_tip_slot_over_lagging_cursor() { + // Cursor persist failures are warn-only, so the read cursor can lag the + // tip by several blocks. The anchor must pair the tip with its own + // inscription slot; pairing it with the stale cursor would make the scan + // misread the chain's intermediate blocks as re-inscriptions. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + + let genesis = common::test_utils::produce_dummy_block(1, None, vec![]); + core.store + .accept_block(&genesis, Slot::from(1_000)) + .await + .expect("accept"); + let block2 = common::test_utils::produce_dummy_block(2, Some(genesis.header.hash), vec![]); + core.store + .accept_block(&block2, Slot::from(1_005)) + .await + .expect("accept"); + let block3 = common::test_utils::produce_dummy_block(3, Some(block2.header.hash), vec![]); + core.store + .accept_block(&block3, Slot::from(1_010)) + .await + .expect("accept"); + + // Cursor last persisted at the genesis slot: two blocks behind the tip. + core.store + .set_zone_cursor(&Slot::from(1_000)) + .expect("set cursor"); + + let anchor = core.get_startup_anchor().expect("anchor").expect("present"); + assert_eq!(anchor.slot, Slot::from(1_010)); + assert_eq!(anchor.block, Some((3, block3.header.hash))); + } + + #[test] + fn probe_finds_anchor_block_at_slot() { + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&tip), Slot::from(1_000)), + AnchorProbe::SameChain + )); + } + + #[test] + fn probe_flags_different_block_at_anchor_id() { + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + // Same id, different content (timestamp) => different hash. + let other = test_block(5, 43); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&other), Slot::from(1_000)), + AnchorProbe::Mismatch(ChainMismatch::Block { .. }) + )); + } + + #[test] + fn probe_flags_old_id_reinscribed_past_the_anchor_slot() { + // A reset chain re-inscribes from genesis at later slots. Even if the + // content is byte-identical (deterministic genesis), an id at/below + // the anchor past the anchor slot is impossible on the same chain. + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + let genesis = test_block(1, 0); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&genesis), Slot::from(1_001)), + AnchorProbe::Mismatch(ChainMismatch::ReinscribedBlock { .. }) + )); + } + + #[test] + fn probe_skips_older_blocks_sharing_the_anchor_slot() { + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + let earlier = test_block(4, 41); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&earlier), Slot::from(1_000)), + AnchorProbe::KeepLooking + )); + } + + #[test] + fn probe_bails_on_newer_ids_past_the_anchor() { + // Blocks newer than the anchor are plausible on the same chain (e.g. + // published while we were down), so they must never count as evidence. + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + let newer = test_block(6, 43); + assert!(matches!( + anchor.probe_anchor_slot(&block_msg(&newer), Slot::from(1_001)), + AnchorProbe::Bail + )); + } + + #[test] + fn probe_skips_undeserializable_inscriptions() { + let tip = test_block(5, 42); + let anchor = anchor_for(&tip, Slot::from(1_000)); + let garbage = ZoneMessage::Block(ZoneBlock { + id: MsgId::from([0_u8; 32]), + data: Inscription::try_from(&[1_u8, 2, 3][..]).expect("inscription"), + }); + assert!(matches!( + anchor.probe_anchor_slot(&garbage, Slot::from(1_000)), + AnchorProbe::KeepLooking + )); + } + + #[test] + fn probe_accepts_any_message_for_a_headerless_anchor() { + // A deserialize park records no header: any message still present at + // the anchor slot means the history is intact. + let anchor = Anchor { + slot: Slot::from(1_000), + block: None, + }; + let garbage = ZoneMessage::Block(ZoneBlock { + id: MsgId::from([0_u8; 32]), + data: Inscription::try_from(&[1_u8, 2, 3][..]).expect("inscription"), + }); + assert!(matches!( + anchor.probe_anchor_slot(&garbage, Slot::from(1_000)), + AnchorProbe::SameChain + )); + assert!(matches!( + anchor.probe_anchor_slot(&garbage, Slot::from(1_001)), + AnchorProbe::Mismatch(ChainMismatch::AnchorSlotChanged { .. }) + )); + } + + #[test] + fn frontier_flags_missing_channel_and_short_history() { + assert!(matches!( + frontier_verdict(Slot::from(1_000), None), + Some(ChainMismatch::ChannelMissing) + )); + assert!(matches!( + frontier_verdict(Slot::from(1_000), Some(Slot::from(999))), + Some(ChainMismatch::ChannelBehindAnchor { .. }) + )); + assert!(frontier_verdict(Slot::from(1_000), Some(Slot::from(1_000))).is_none()); + assert!(frontier_verdict(Slot::from(1_000), Some(Slot::from(2_000))).is_none()); + } +} diff --git a/lez/indexer/core/src/config.rs b/lez/indexer/core/src/config.rs index f920fd5b..159da23c 100644 --- a/lez/indexer/core/src/config.rs +++ b/lez/indexer/core/src/config.rs @@ -30,6 +30,13 @@ pub struct IndexerConfig { /// transaction, so the indexer must seed them to match the sequencer's state. #[serde(default)] pub bridge_lock_holdings: Vec, + /// Whether to wipe the indexer store and re-index from scratch when the startup + /// chain-identity check finds the channel serving a different block than the one + /// stored at the same id. + /// + /// Defaults to `false`: on mismatch the indexer refuses to start. + #[serde(default)] + pub allow_chain_reset: bool, } /// A genesis-funded bridge-lock holder balance, configured identically on the diff --git a/lez/indexer/core/src/ingest_error.rs b/lez/indexer/core/src/ingest_error.rs new file mode 100644 index 00000000..299013d7 --- /dev/null +++ b/lez/indexer/core/src/ingest_error.rs @@ -0,0 +1,80 @@ +use common::HashType; +use serde::{Deserialize, Serialize}; + +/// Why the indexer could not apply an L2 block from the channel. +/// +/// Persisted in `RocksDB`, so every variant must have the following +/// traits: `Clone + Serialize + Deserialize`. +#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] +pub enum BlockIngestError { + #[error("Failed to deserialize L2 block: {0}")] + /// Here we store the error string that is derived from [`borsh::from_slice`]'s [`Err`]. + Deserialize(String), + #[error("Unexpected block id: expected {expected}, got {got}")] + UnexpectedBlockId { expected: u64, got: u64 }, + #[error("Broken chain link: expected prev {expected_prev}, got {got_prev}")] + BrokenChainLink { + expected_prev: HashType, + got_prev: HashType, + }, + #[error("Block hash mismatch: computed {computed}, header {header}")] + HashMismatch { + computed: HashType, + header: HashType, + }, + #[error("Block has no transactions")] + EmptyBlock, + #[error("Last transaction must be the public clock invocation for the block timestamp")] + InvalidClockTransaction, + #[error("Genesis block must contain only public transactions")] + NonPublicGenesisTransaction, + #[error("State transition failed at transaction {tx_index}: {reason}")] + StateTransition { + /// Index of the failing transaction within the block body. + tx_index: u64, + /// Reason string from `lee::Error` to `anyhow::Error` to `{:#}`. + /// + /// This is required because `lee::Error` is not `Clone + Serialize + Deserialize`, so we + /// cannot store it directly. + reason: String, + }, +} + +impl BlockIngestError { + /// Whether the failure may be transient rather than a property of the block. + /// + /// FIXME: `StateTransition` is too coarse โ€” its `reason` string mixes genuine + /// state-transition rejections with infra failures (risc0 executor teardown, + /// storage errors). Once it carries a structured cause, narrow this so only + /// infra failures retry. + #[must_use] + pub const fn is_retryable(&self) -> bool { + matches!(self, Self::StateTransition { .. }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn serializes_and_round_trips_externally_tagged() { + let err = BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7, + }; + let value = serde_json::to_value(&err).expect("serialize"); + assert_eq!( + value, + serde_json::json!({ "UnexpectedBlockId": { "expected": 5, "got": 7 } }) + ); + let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); + assert!(matches!( + back, + BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7 + } + )); + } +} diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index e7f6460a..7bee6387 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -3,29 +3,40 @@ use std::{path::Path, sync::Arc}; use anyhow::Result; use arc_swap::ArcSwap; use common::block::Block; -// ToDo: Remove after testnet +// TODO: Remove after testnet use futures::StreamExt as _; +pub use ingest_error::BlockIngestError; use log::{error, info, warn}; -use logos_blockchain_core::header::HeaderId; use logos_blockchain_zone_sdk::{ - CommonHttpClient, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, + CommonHttpClient, Slot, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, }; +use retry::ApplyRetryGate; +pub use stall_reason::StallReason; use crate::{ - block_store::IndexerStore, + block_store::{AcceptOutcome, IndexerStore}, + chain_consistency::ChainConsistency, config::IndexerConfig, cross_zone_verifier::CrossZoneVerifier, status::{IndexerStatus, IndexerSyncStatus}, }; - pub mod block_store; +pub mod chain_consistency; pub mod config; pub mod cross_zone_verifier; +pub mod ingest_error; +mod retry; +pub mod stall_reason; pub mod status; +/// Consecutive failed apply attempts of the same block before parking. +const APPLY_RETRY_LIMIT: u32 = 3; + #[derive(Clone)] pub struct IndexerCore { pub zone_indexer: Arc>, + /// Direct node handle for queries outside `ZoneIndexer`'s streaming API. + pub node: NodeHttpClient, pub config: IndexerConfig, pub store: IndexerStore, /// Live ingestion status; updated by the ingest stream, read by `status`. @@ -35,7 +46,42 @@ pub struct IndexerCore { } impl IndexerCore { - pub fn new(config: IndexerConfig, storage_dir: &Path) -> Result { + /// Builds the core, then verifies the stored chain matches the channel's by + /// re-reading the channel at the stored tip's position. + /// + /// On mismatch: refuse (error) unless `config.allow_chain_reset` is set, in which case wipe the + /// store and re-index from scratch. + pub async fn new(config: IndexerConfig, storage_dir: &Path) -> Result { + let home = storage_dir.join(format!("rocksdb-{}", config.channel_id)); + let core = Self::open(config.clone(), storage_dir)?; + match core.verify_chain_consistency().await? { + // `Inconclusive` is deliberately treated the same as `Consistent`. + // + // We could not prove a reset, so proceed from the cursor without wiping + // a possibly-valid store. A genuinely divergent chain is still caught + // later when the ingest loop tries to apply and parks. + ChainConsistency::Consistent | ChainConsistency::Inconclusive => Ok(core), + ChainConsistency::Inconsistent(mismatch) if config.allow_chain_reset => { + warn!( + "Chain reset detected ({mismatch}). Wiping indexer store at {} and \ + re-indexing.", + home.display() + ); + drop(core); // sole owner before the ingest task is spawned โ†’ closes the DB + storage::indexer::RocksDBIO::destroy(&home)?; + Self::open(config, storage_dir) + } + ChainConsistency::Inconsistent(mismatch) => Err(anyhow::anyhow!( + "Indexer store at {} holds a different chain than the channel now serves \ + ({mismatch}). Delete the indexer storage directory, point at a fresh one, or \ + set `allow_chain_reset` in the indexer config.", + home.display() + )), + } + } + + /// Opens the store and builds the core without the chain-identity check. + fn open(config: IndexerConfig, storage_dir: &Path) -> Result { // Namespace the DB by channel so indexers on different channels can // share a storage dir without their RocksDB state colliding. let home = storage_dir.join(format!("rocksdb-{}", config.channel_id)); @@ -45,7 +91,7 @@ impl IndexerCore { CommonHttpClient::new(basic_auth), config.bedrock_config.addr.clone(), ); - let zone_indexer = ZoneIndexer::new(config.channel_id, node); + let zone_indexer = ZoneIndexer::new(config.channel_id, node.clone()); // Genesis accounts the indexer must seed to match the sequencer's state, // since none are produced by a transaction: the cross-zone inbox config @@ -72,6 +118,7 @@ impl IndexerCore { Ok(Self { zone_indexer: Arc::new(zone_indexer), store: IndexerStore::open_db(&home, genesis_seed)?, + node, config, status: Arc::new(ArcSwap::from_pointee(IndexerSyncStatus::starting())), verifier, @@ -85,10 +132,27 @@ impl IndexerCore { #[must_use] pub fn status(&self) -> IndexerStatus { let sync = IndexerSyncStatus::clone(&self.status.load()); - let indexed_block_id = self.store.get_last_block_id().ok().flatten(); + // Log-and-fall-back rather than collapsing a store error into the same + // `None` as "legitimately absent": a DB read failure must not silently + // masquerade as "no tip yet" / "no stall recorded" in the snapshot. + let indexed_block_id = match self.store.get_last_block_id() { + Ok(id) => id, + Err(err) => { + warn!("Failed to read last indexed block id for status: {err:#}"); + None + } + }; + let stall_reason = match self.store.get_stall_reason() { + Ok(reason) => reason, + Err(err) => { + warn!("Failed to read stall reason for status: {err:#}"); + None + } + }; IndexerStatus { sync, indexed_block_id, + stall_reason, } } @@ -97,6 +161,41 @@ impl IndexerCore { self.status.store(Arc::new(status)); } + /// Advances the in-memory L1 read cursor past `slot` and persists it. + /// A persist failure is only logged: the worst case is re-reading a batch + /// after a restart, which ingestion handles idempotently. + fn advance_cursor(&self, cursor: &mut Option, slot: Slot) { + *cursor = Some(slot); + if let Err(err) = self.store.set_zone_cursor(&slot) { + warn!("Failed to persist indexer cursor: {err:#}"); + } + } + + /// Parks on an inscription that could not be parsed as an L2 block: + /// records the stall and flips the status. The validated tip stays frozen. + /// + /// Returns `false` if the stall could not be recorded durably; the caller + /// must then hold the cursor and retry instead of advancing past the slot. + fn park_undeserializable(&self, slot: Slot, error: std::io::Error) -> bool { + let error = anyhow::Error::new(error); + + // use `:#` to get the entire error chain + let reason = format!("{error:#}"); + error!("Failed to deserialize L2 block from zone-sdk: {reason}"); + if let Err(err) = + self.store + .record_stall(None, slot, BlockIngestError::Deserialize(reason.clone())) + { + error!("Failed to record stall reason: {err:#}"); + self.set_status(IndexerSyncStatus::error(format!("store error: {err:#}"))); + return false; + } + self.set_status(IndexerSyncStatus::stalled(format!( + "failed to deserialize L2 block: {reason}" + ))); + true + } + pub fn subscribe_parse_block_stream(&self) -> impl futures::Stream> + '_ { let poll_interval = self.config.consensus_info_polling_interval; let initial_cursor = self @@ -106,6 +205,7 @@ impl IndexerCore { async_stream::stream! { let mut cursor = initial_cursor; + let mut retry_gate = ApplyRetryGate::new(); if cursor.is_some() { info!("Resuming indexer from cursor {cursor:?}"); @@ -117,8 +217,6 @@ impl IndexerCore { let stream = match self.zone_indexer.next_messages(cursor).await { Ok(s) => s, Err(err) => { - // `next_messages` reads L1 consensus info internally, so - // this also covers an unreachable/misconfigured L1 node. error!("Failed to start zone-sdk next_messages stream: {err}"); self.set_status(IndexerSyncStatus::error(format!( "cannot reach L1 / read channel: {err}" @@ -129,11 +227,8 @@ impl IndexerCore { }; let mut stream = std::pin::pin!(stream); - // Flip to Syncing on the first message of this cycle (not merely on - // a successful poll) so the steady-state CaughtUp status doesn't - // flicker. Until then the state stays Starting (cold-start scan of - // empty L1 history) or CaughtUp (idle). let mut announced_syncing = false; + let mut had_cycle_error = false; while let Some((msg, slot)) = stream.next().await { if !announced_syncing { @@ -143,27 +238,24 @@ impl IndexerCore { let zone_block = match msg { ZoneMessage::Block(b) => b, - // Non-block messages don't carry a cursor position; the - // next ZoneBlock advances past them implicitly. + // FIXME: will be handled in prep of decentralized sequencers ZoneMessage::Deposit(_) | ZoneMessage::Withdraw(_) => continue, }; let block: Block = match borsh::from_slice(&zone_block.data) { Ok(b) => b, - Err(e) => { - error!("Failed to deserialize L2 block from zone-sdk: {e}"); - // Advance past the broken inscription so we don't - // re-process it on restart. - cursor = Some(slot); - if let Err(err) = self.store.set_zone_cursor(&slot) { - warn!("Failed to persist indexer cursor: {err:#}"); + Err(error) => { + // The stall must be durable before the cursor moves. + if !self.park_undeserializable(slot, error) { + had_cycle_error = true; + break; } + // L1 proceeds regardless + self.advance_cursor(&mut cursor, slot); continue; } }; - info!("Indexed L2 block {}", block.header.block_id); - // Option B: re-derive and verify every cross-zone dispatch // before applying the block. A forged or replayed dispatch // halts ingestion rather than persisting an invalid state. @@ -174,34 +266,105 @@ impl IndexerCore { "Cross-zone verification failed for block {}: {err:#}. Halting indexer ingestion.", block.header.block_id ); + self.set_status(IndexerSyncStatus::error(format!( + "cross-zone verification failed: {err:#}" + ))); return; } - // TODO: Remove l1_header placeholder once storage layer - // no longer requires it. Zone-sdk handles L1 tracking internally. - let placeholder_l1_header = HeaderId::from([0_u8; 32]); - if let Err(err) = self.store.put_block(block.clone(), placeholder_l1_header).await { - // Do not advance the cursor past a block we failed to - // apply: halt ingestion instead of silently desyncing. - error!( - "Failed to store block {}: {err:#}. Halting indexer ingestion.", - block.header.block_id - ); - return; + match self.store.accept_block(&block, slot).await { + Ok(AcceptOutcome::Applied) => { + retry_gate.reset(); + info!("Indexed L2 block {}", block.header.block_id); + self.set_status(IndexerSyncStatus::syncing()); + self.advance_cursor(&mut cursor, slot); + yield Ok(block); + } + Ok(AcceptOutcome::AlreadyApplied) => { + info!( + "Skipping already-applied block {}", + block.header.block_id + ); + self.advance_cursor(&mut cursor, slot); + } + Ok(AcceptOutcome::Parked(ingest_err)) => { + error!( + "Parked at block {}: {ingest_err}", + block.header.block_id + ); + self.set_status(IndexerSyncStatus::stalled(ingest_err.to_string())); + // L1 proceeds regardless + self.advance_cursor(&mut cursor, slot); + } + Ok(AcceptOutcome::RetryableFailure(ingest_err)) => { + let attempts = retry_gate.register_failure(block.header.block_id); + if attempts >= APPLY_RETRY_LIMIT { + error!( + "Parked at block {} after {attempts} failed apply attempts: {ingest_err}", + block.header.block_id + ); + // The stall must be durable before the cursor moves. + if let Err(err) = self.store.record_stall( + Some(&block.header), + slot, + ingest_err.clone(), + ) { + error!( + "Failed to record stall reason for block {}: {err:#}", + block.header.block_id + ); + self.set_status(IndexerSyncStatus::error(format!( + "store error: {err:#}" + ))); + had_cycle_error = true; + break; + } + self.set_status(IndexerSyncStatus::stalled(ingest_err.to_string())); + self.advance_cursor(&mut cursor, slot); + retry_gate.reset(); + } else { + error!( + "Failed to apply block {} (attempt {attempts}/{APPLY_RETRY_LIMIT}), will retry: {ingest_err}", + block.header.block_id + ); + self.set_status(IndexerSyncStatus::error(format!( + "apply failed, retrying: {ingest_err}" + ))); + had_cycle_error = true; + break; + } + } + Err(err) => { + // Infrastructure error (DB read/write), not a bad block. + // will re-poll from the same cursor next cycle. + error!( + "Store error applying block {}: {err:#}", + block.header.block_id + ); + self.set_status(IndexerSyncStatus::error(format!( + "store error: {err:#}" + ))); + had_cycle_error = true; + break; + } } - - cursor = Some(slot); - if let Err(err) = self.store.set_zone_cursor(&slot) { - warn!("Failed to persist indexer cursor: {err:#}"); - } - yield Ok(block); } - // Stream drained: caught up to LIB as of this cycle. Clears any - // prior error (e.g. a transient L1 disconnect that left no - // backlog, so the `Syncing` branch above never ran). Sleep then - // poll again. - self.set_status(IndexerSyncStatus::caught_up()); + if had_cycle_error { + tokio::time::sleep(poll_interval).await; + continue; + } + + // Stream drained. Stay Stalled if parked; otherwise we are caught up. + // A store error here must not be collapsed to "no stall recorded": + // that would wrongly flip us to caught-up, so we log and hold state. + match self.store.get_stall_reason() { + Ok(None) => self.set_status(IndexerSyncStatus::caught_up()), + Ok(Some(_)) => {} + Err(err) => { + warn!("Failed to read stall reason after draining stream; not marking caught up: {err:#}"); + } + } tokio::time::sleep(poll_interval).await; } } diff --git a/lez/indexer/core/src/retry.rs b/lez/indexer/core/src/retry.rs new file mode 100644 index 00000000..d7c5f5a2 --- /dev/null +++ b/lez/indexer/core/src/retry.rs @@ -0,0 +1,64 @@ +//! Retry gate for possibly-transient block-apply failures. + +/// Counts consecutive apply failures per block id so the ingest loop can +/// retry before parking. +/// +/// A failure streak is keyed to one block id: a failure of a different block +/// starts a fresh streak. Reset only on a genuinely applied block โ€” the +/// `AlreadyApplied` replay of the prefix after a retry cycle must not clear +/// the failing block's streak. +pub struct ApplyRetryGate { + failing: Option<(u64, u32)>, +} + +impl ApplyRetryGate { + #[must_use] + pub const fn new() -> Self { + Self { failing: None } + } + + /// Registers a failed apply of `block_id`; returns its consecutive + /// attempt count. + pub const fn register_failure(&mut self, block_id: u64) -> u32 { + let attempts = match self.failing { + Some((id, attempts)) if id == block_id => attempts.saturating_add(1), + _ => 1, + }; + self.failing = Some((block_id, attempts)); + attempts + } + + /// Clears the streak; call when a block actually applies. + pub const fn reset(&mut self) { + self.failing = None; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counts_consecutive_failures_of_same_block() { + let mut gate = ApplyRetryGate::new(); + assert_eq!(gate.register_failure(7), 1); + assert_eq!(gate.register_failure(7), 2); + assert_eq!(gate.register_failure(7), 3); + } + + #[test] + fn different_block_starts_fresh_streak() { + let mut gate = ApplyRetryGate::new(); + gate.register_failure(7); + gate.register_failure(7); + assert_eq!(gate.register_failure(8), 1); + } + + #[test] + fn reset_clears_streak() { + let mut gate = ApplyRetryGate::new(); + gate.register_failure(7); + gate.reset(); + assert_eq!(gate.register_failure(7), 1); + } +} diff --git a/lez/indexer/core/src/stall_reason.rs b/lez/indexer/core/src/stall_reason.rs new file mode 100644 index 00000000..c5118fcc --- /dev/null +++ b/lez/indexer/core/src/stall_reason.rs @@ -0,0 +1,25 @@ +use common::HashType; +use logos_blockchain_zone_sdk::Slot; +use serde::{Deserialize, Serialize}; + +use crate::ingest_error::BlockIngestError; + +/// Diagnostic record of the first block that broke the L2 chain. +/// +/// The block-derived fields are `None` for a deserialize break (no header was +/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. +/// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StallReason { + pub block_id: Option, + pub block_hash: Option, + pub prev_block_hash: Option, + pub l1_slot: Slot, + pub error: BlockIngestError, + pub first_seen: Option, + /// Number of later non-chaining blocks (orphans, since the tip is frozen). + /// + /// TODO: We could store a different "branch" of blocks following this break, but for now we + /// just count them. + pub orphans_since: u64, +} diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index 1193e124..a483fde6 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -1,9 +1,10 @@ use serde::Serialize; +use crate::stall_reason::StallReason; + /// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell /// "still catching up" apart from "something went wrong". #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] -#[serde(rename_all = "snake_case")] pub enum IndexerSyncState { /// Booted; no ingestion cycle has run yet. Starting, @@ -13,12 +14,14 @@ pub enum IndexerSyncState { CaughtUp, /// The last cycle failed (e.g. the L1 node is unreachable). See `last_error`. Error, + /// Parked on a stall reason: the validated tip is frozen awaiting a valid + /// continuation. See `last_error` and the snapshot's `stall_reason`. + Stalled, } /// Live ingestion status owned by the ingest loop: the coarse `state` plus the /// reason when it is `Error`. #[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] pub struct IndexerSyncStatus { pub state: IndexerSyncState, pub last_error: Option, @@ -56,6 +59,15 @@ impl IndexerSyncStatus { last_error: Some(reason), } } + + /// Parked on a stall reason; `reason` mirrors the stall's error message. + /// The full stall is attached to the [`IndexerStatus`] snapshot. + pub(crate) const fn stalled(reason: String) -> Self { + Self { + state: IndexerSyncState::Stalled, + last_error: Some(reason), + } + } } /// Full status snapshot returned to callers (FFI/RPC): the live [`IndexerSyncStatus`] @@ -64,11 +76,11 @@ impl IndexerSyncStatus { /// The tip is tracked by the store, not the ingest loop, so it lives here on the /// returned snapshot rather than inside the shared [`IndexerSyncStatus`]. #[derive(Debug, Clone, Serialize)] -#[serde(rename_all = "camelCase")] pub struct IndexerStatus { #[serde(flatten)] pub sync: IndexerSyncStatus, pub indexed_block_id: Option, + pub stall_reason: Option, } #[cfg(test)] @@ -80,14 +92,16 @@ mod tests { let status = IndexerStatus { sync: IndexerSyncStatus::error("boom".to_owned()), indexed_block_id: Some(7), + stall_reason: None, }; let value = serde_json::to_value(&status).expect("serialize"); assert_eq!( value, serde_json::json!({ - "state": "error", - "lastError": "boom", - "indexedBlockId": 7, + "state": "Error", + "last_error": "boom", + "indexed_block_id": 7, + "stall_reason": null, }) ); } @@ -97,7 +111,36 @@ mod tests { let value = serde_json::to_value(IndexerSyncStatus::caught_up()).expect("serialize"); assert_eq!( value, - serde_json::json!({ "state": "caught_up", "lastError": null }) + serde_json::json!({ "state": "CaughtUp", "last_error": null }) ); } + + #[test] + fn stalled_status_serializes_with_stall_reason() { + use logos_blockchain_zone_sdk::Slot; + + use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; + + let status = IndexerStatus { + sync: IndexerSyncStatus::stalled("broken chain link".to_owned()), + indexed_block_id: Some(41), + stall_reason: Some(StallReason { + block_id: Some(42), + block_hash: None, + prev_block_hash: None, + l1_slot: Slot::from(0), + error: BlockIngestError::StateTransition { + tx_index: 0, + reason: String::default(), + }, + first_seen: None, + orphans_since: 2, + }), + }; + let value = serde_json::to_value(&status).expect("serialize"); + assert_eq!(value["state"], serde_json::json!("Stalled")); + assert_eq!(value["last_error"], serde_json::json!("broken chain link")); + assert_eq!(value["indexed_block_id"], serde_json::json!(41)); + assert_eq!(value["stall_reason"]["orphans_since"], serde_json::json!(2)); + } } diff --git a/lez/indexer/ffi/indexer_ffi.h b/lez/indexer/ffi/indexer_ffi.h index 8347ad3c..26857af7 100644 --- a/lez/indexer/ffi/indexer_ffi.h +++ b/lez/indexer/ffi/indexer_ffi.h @@ -503,9 +503,9 @@ struct LastBlockIdResult query_last_block(const struct IndexerServiceFFI *indexe * Query the indexer's current sync status as a JSON C-string. * * The JSON schema is owned by `indexer_core` (`IndexerStatus`): an object with - * `state` (`starting`/`syncing`/`caught_up`/`error`), `indexedBlockId`, and - * `lastError`. Lets a client distinguish "still catching up" from "something - * went wrong". + * `state` (`Starting`/`Syncing`/`CaughtUp`/`Error`/`Stalled`), + * `indexed_block_id`, `last_error`, and `stall_reason`. Lets a client + * distinguish "still catching up" from "something went wrong". * * # Arguments * diff --git a/lez/indexer/ffi/src/api/lifecycle.rs b/lez/indexer/ffi/src/api/lifecycle.rs index f668f3ee..8a1eafaf 100644 --- a/lez/indexer/ffi/src/api/lifecycle.rs +++ b/lez/indexer/ffi/src/api/lifecycle.rs @@ -112,10 +112,12 @@ unsafe fn setup_indexer( unsafe { Runtime::from_borrowed(caller.as_ref()) } }; - let core = IndexerCore::new(config, &storage_dir).map_err(|e| { - log::error!("Could not initialize indexer core: {e}"); - OperationStatus::InitializationError - })?; + let core = runtime + .block_on(IndexerCore::new(config, &storage_dir)) + .map_err(|e| { + log::error!("Could not initialize indexer core: {e}"); + OperationStatus::InitializationError + })?; // The block stream writes each parsed block into the store as a side effect // of being polled, so we spawn a task that simply drains it. There are no diff --git a/lez/indexer/ffi/src/api/query.rs b/lez/indexer/ffi/src/api/query.rs index 1943f6d4..dff027fa 100644 --- a/lez/indexer/ffi/src/api/query.rs +++ b/lez/indexer/ffi/src/api/query.rs @@ -91,9 +91,9 @@ pub unsafe extern "C" fn query_last_block(indexer: *const IndexerServiceFFI) -> /// Query the indexer's current sync status as a JSON C-string. /// /// The JSON schema is owned by `indexer_core` (`IndexerStatus`): an object with -/// `state` (`starting`/`syncing`/`caught_up`/`error`), `indexedBlockId`, and -/// `lastError`. Lets a client distinguish "still catching up" from "something -/// went wrong". +/// `state` (`Starting`/`Syncing`/`CaughtUp`/`Error`/`Stalled`), +/// `indexed_block_id`, `last_error`, and `stall_reason`. Lets a client +/// distinguish "still catching up" from "something went wrong". /// /// # Arguments /// diff --git a/lez/indexer/service/configs/debug/indexer_config.json b/lez/indexer/service/configs/debug/indexer_config.json index 85227700..be5cf353 100644 --- a/lez/indexer/service/configs/debug/indexer_config.json +++ b/lez/indexer/service/configs/debug/indexer_config.json @@ -1,7 +1,8 @@ { - "consensus_info_polling_interval": "1s", - "bedrock_config": { - "addr": "http://localhost:18080" - }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101" + "consensus_info_polling_interval": "1s", + "bedrock_config": { + "addr": "http://localhost:18080" + }, + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "allow_chain_reset": true } diff --git a/lez/indexer/service/configs/docker/indexer_config.json b/lez/indexer/service/configs/docker/indexer_config.json index f083ca27..ce28af0b 100644 --- a/lez/indexer/service/configs/docker/indexer_config.json +++ b/lez/indexer/service/configs/docker/indexer_config.json @@ -3,5 +3,6 @@ "bedrock_config": { "addr": "http://host.docker.internal:18080" }, - "channel_id": "0101010101010101010101010101010101010101010101010101010101010101" + "channel_id": "0101010101010101010101010101010101010101010101010101010101010101", + "allow_chain_reset": true } diff --git a/lez/indexer/service/protocol/Cargo.toml b/lez/indexer/service/protocol/Cargo.toml index 5a4176f5..b3e8f65c 100644 --- a/lez/indexer/service/protocol/Cargo.toml +++ b/lez/indexer/service/protocol/Cargo.toml @@ -11,6 +11,7 @@ workspace = true lee_core = { workspace = true, optional = true, features = ["host"] } lee = { workspace = true, optional = true } common = { workspace = true, optional = true } +indexer_core = { workspace = true, optional = true } serde = { workspace = true, features = ["derive"] } serde_with.workspace = true @@ -22,4 +23,4 @@ anyhow.workspace = true [features] # Enable conversion to/from LEE core types -convert = ["dep:lee_core", "dep:lee", "dep:common"] +convert = ["dep:lee_core", "dep:lee", "dep:common", "dep:indexer_core"] diff --git a/lez/indexer/service/protocol/src/convert.rs b/lez/indexer/service/protocol/src/convert.rs index cd0bff7e..55c4dc6c 100644 --- a/lez/indexer/service/protocol/src/convert.rs +++ b/lez/indexer/service/protocol/src/convert.rs @@ -3,11 +3,12 @@ use lee_core::account::Nonce; use crate::{ - Account, AccountId, BedrockStatus, Block, BlockBody, BlockHeader, Ciphertext, Commitment, - CommitmentSetDigest, Data, EncryptedAccountData, EphemeralPublicKey, HashType, Nullifier, - PrivacyPreservingMessage, PrivacyPreservingTransaction, ProgramDeploymentMessage, - ProgramDeploymentTransaction, ProgramId, Proof, PublicKey, PublicMessage, PublicTransaction, - Signature, Transaction, ValidityWindow, WitnessSet, + Account, AccountId, BedrockStatus, Block, BlockBody, BlockHeader, BlockIngestError, Ciphertext, + Commitment, CommitmentSetDigest, Data, EncryptedAccountData, EphemeralPublicKey, HashType, + IndexerStatus, IndexerSyncState, Nullifier, PrivacyPreservingMessage, + PrivacyPreservingTransaction, ProgramDeploymentMessage, ProgramDeploymentTransaction, + ProgramId, Proof, PublicKey, PublicMessage, PublicTransaction, Signature, StallReason, + Transaction, ValidityWindow, WitnessSet, }; // ============================================================================ @@ -707,3 +708,94 @@ impl TryFrom for lee_core::program::ValidityWindow { value.0.try_into() } } + +// ============================================================================ +// Indexer status conversions +// ============================================================================ + +impl From for IndexerSyncState { + fn from(value: indexer_core::status::IndexerSyncState) -> Self { + match value { + indexer_core::status::IndexerSyncState::Starting => Self::Starting, + indexer_core::status::IndexerSyncState::Syncing => Self::Syncing, + indexer_core::status::IndexerSyncState::CaughtUp => Self::CaughtUp, + indexer_core::status::IndexerSyncState::Error => Self::Error, + indexer_core::status::IndexerSyncState::Stalled => Self::Stalled, + } + } +} + +impl From for BlockIngestError { + fn from(value: indexer_core::BlockIngestError) -> Self { + match value { + indexer_core::BlockIngestError::Deserialize(msg) => Self::Deserialize(msg), + indexer_core::BlockIngestError::UnexpectedBlockId { expected, got } => { + Self::UnexpectedBlockId { expected, got } + } + indexer_core::BlockIngestError::BrokenChainLink { + expected_prev, + got_prev, + } => Self::BrokenChainLink { + expected_prev: expected_prev.into(), + got_prev: got_prev.into(), + }, + indexer_core::BlockIngestError::HashMismatch { computed, header } => { + Self::HashMismatch { + computed: computed.into(), + header: header.into(), + } + } + indexer_core::BlockIngestError::EmptyBlock => Self::EmptyBlock, + indexer_core::BlockIngestError::InvalidClockTransaction => { + Self::InvalidClockTransaction + } + indexer_core::BlockIngestError::NonPublicGenesisTransaction => { + Self::NonPublicGenesisTransaction + } + indexer_core::BlockIngestError::StateTransition { tx_index, reason } => { + Self::StateTransition { tx_index, reason } + } + } + } +} + +impl From for StallReason { + fn from(value: indexer_core::StallReason) -> Self { + let indexer_core::StallReason { + block_id, + block_hash, + prev_block_hash, + l1_slot, + error, + first_seen, + orphans_since, + } = value; + + Self { + block_id, + block_hash: block_hash.map(Into::into), + prev_block_hash: prev_block_hash.map(Into::into), + l1_slot: l1_slot.into_inner(), + error: error.into(), + first_seen, + orphans_since, + } + } +} + +impl From for IndexerStatus { + fn from(value: indexer_core::status::IndexerStatus) -> Self { + let indexer_core::status::IndexerStatus { + sync, + indexed_block_id, + stall_reason, + } = value; + + Self { + state: sync.state.into(), + last_error: sync.last_error, + indexed_block_id, + stall_reason: stall_reason.map(Into::into), + } + } +} diff --git a/lez/indexer/service/protocol/src/lib.rs b/lez/indexer/service/protocol/src/lib.rs index a670dee6..e17d539b 100644 --- a/lez/indexer/service/protocol/src/lib.rs +++ b/lez/indexer/service/protocol/src/lib.rs @@ -363,3 +363,73 @@ pub enum BedrockStatus { Safe, Finalized, } + +/// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell +/// "still catching up" apart from "something went wrong". +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub enum IndexerSyncState { + /// Booted; no ingestion cycle has run yet. + Starting, + /// Streaming finalized messages toward the L1 frontier. + Syncing, + /// Drained the stream up to the last finalized block; idle until new blocks finalize. + CaughtUp, + /// The last cycle failed (e.g. the L1 node is unreachable). See `last_error`. + Error, + /// Parked on a stall reason: the validated tip is frozen awaiting a valid + /// continuation. See `last_error` and `stall_reason`. + Stalled, +} + +/// Why the indexer could not apply an L2 block from the channel. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub enum BlockIngestError { + Deserialize(String), + UnexpectedBlockId { + expected: u64, + got: u64, + }, + BrokenChainLink { + expected_prev: HashType, + got_prev: HashType, + }, + HashMismatch { + computed: HashType, + header: HashType, + }, + EmptyBlock, + InvalidClockTransaction, + NonPublicGenesisTransaction, + StateTransition { + /// Index of the failing transaction within the block body. + tx_index: u64, + reason: String, + }, +} + +/// Diagnostic record of the first block that broke the L2 chain. +/// +/// The block-derived fields are `None` for a deserialize break (no header was +/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub struct StallReason { + pub block_id: Option, + pub block_hash: Option, + pub prev_block_hash: Option, + pub l1_slot: u64, + pub error: BlockIngestError, + /// The breaking block's L2 timestamp (`None` for a deserialize break). + pub first_seen: Option, + /// Number of later non-chaining blocks seen while the tip is frozen. + pub orphans_since: u64, +} + +/// Status snapshot returned by `getStatus`: the ingestion state plus the +/// indexed L2 tip and, when stalled, the stall diagnostics. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +pub struct IndexerStatus { + pub state: IndexerSyncState, + pub last_error: Option, + pub indexed_block_id: Option, + pub stall_reason: Option, +} diff --git a/lez/indexer/service/rpc/src/lib.rs b/lez/indexer/service/rpc/src/lib.rs index 5763fe82..8ea807eb 100644 --- a/lez/indexer/service/rpc/src/lib.rs +++ b/lez/indexer/service/rpc/src/lib.rs @@ -1,4 +1,6 @@ -use indexer_service_protocol::{Account, AccountId, Block, BlockId, HashType, Transaction}; +use indexer_service_protocol::{ + Account, AccountId, Block, BlockId, HashType, IndexerStatus, Transaction, +}; use jsonrpsee::proc_macros::rpc; #[cfg(feature = "server")] use jsonrpsee::{core::SubscriptionResult, types::ErrorObjectOwned}; @@ -69,6 +71,9 @@ pub trait Rpc { limit: u64, ) -> Result, ErrorObjectOwned>; + #[method(name = "getStatus")] + async fn get_status(&self) -> Result; + // ToDo: expand healthcheck response into some kind of report #[method(name = "checkHealth")] async fn healthcheck(&self) -> Result<(), ErrorObjectOwned>; diff --git a/lez/indexer/service/src/lib.rs b/lez/indexer/service/src/lib.rs index b1c57163..aa142b38 100644 --- a/lez/indexer/service/src/lib.rs +++ b/lez/indexer/service/src/lib.rs @@ -5,6 +5,7 @@ pub use indexer_core::config::*; use indexer_service_rpc::RpcServer as _; use jsonrpsee::server::{Server, ServerHandle}; use log::{error, info}; +use tokio_util::sync::CancellationToken; pub mod service; @@ -69,9 +70,10 @@ pub async fn run_server( config: IndexerConfig, storage_dir: &Path, port: u16, + shutdown: CancellationToken, ) -> Result { #[cfg(feature = "mock-responses")] - let _ = (config, storage_dir); + let _ = (config, storage_dir, shutdown); let server = Server::builder() .build(SocketAddr::from(([0, 0, 0, 0], port))) @@ -86,7 +88,8 @@ pub async fn run_server( #[cfg(not(feature = "mock-responses"))] let handle = { - let service = service::IndexerService::new(config, storage_dir) + let service = service::IndexerService::new(config, storage_dir, shutdown.child_token()) + .await .context("Failed to initialize indexer service")?; server.start(service.into_rpc()) }; diff --git a/lez/indexer/service/src/main.rs b/lez/indexer/service/src/main.rs index 3e36967d..52f195e9 100644 --- a/lez/indexer/service/src/main.rs +++ b/lez/indexer/service/src/main.rs @@ -34,7 +34,9 @@ async fn main() -> Result<()> { let cancellation_token = listen_for_shutdown_signal(); let config = indexer_service::IndexerConfig::from_path(&config_path)?; - let indexer_handle = indexer_service::run_server(config, data_dir.as_path(), port).await?; + let indexer_handle = + indexer_service::run_server(config, data_dir.as_path(), port, cancellation_token.clone()) + .await?; tokio::select! { () = cancellation_token.cancelled() => { diff --git a/lez/indexer/service/src/mock_service.rs b/lez/indexer/service/src/mock_service.rs index 7bf6c528..70af6239 100644 --- a/lez/indexer/service/src/mock_service.rs +++ b/lez/indexer/service/src/mock_service.rs @@ -10,10 +10,10 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use indexer_service_protocol::{ Account, AccountId, BedrockStatus, Block, BlockBody, BlockHeader, BlockId, Commitment, - CommitmentSetDigest, Data, EncryptedAccountData, HashType, PrivacyPreservingMessage, - PrivacyPreservingTransaction, ProgramDeploymentMessage, ProgramDeploymentTransaction, - ProgramId, PublicMessage, PublicTransaction, Signature, Transaction, ValidityWindow, - WitnessSet, + CommitmentSetDigest, Data, EncryptedAccountData, HashType, IndexerStatus, IndexerSyncState, + PrivacyPreservingMessage, PrivacyPreservingTransaction, ProgramDeploymentMessage, + ProgramDeploymentTransaction, ProgramId, PublicMessage, PublicTransaction, Signature, + Transaction, ValidityWindow, WitnessSet, }; use jsonrpsee::{ core::{SubscriptionResult, async_trait}, @@ -325,6 +325,24 @@ impl indexer_service_rpc::RpcServer for MockIndexerService { .collect()) } + async fn get_status(&self) -> Result { + let indexed_block_id = self + .state + .read() + .await + .blocks + .iter() + .rev() + .find(|block| block.bedrock_status == BedrockStatus::Finalized) + .map(|block| block.header.block_id); + Ok(IndexerStatus { + state: IndexerSyncState::CaughtUp, + last_error: None, + indexed_block_id, + stall_reason: None, + }) + } + async fn healthcheck(&self) -> Result<(), ErrorObjectOwned> { Ok(()) } diff --git a/lez/indexer/service/src/service.rs b/lez/indexer/service/src/service.rs index 7a8ed90f..09759362 100644 --- a/lez/indexer/service/src/service.rs +++ b/lez/indexer/service/src/service.rs @@ -2,9 +2,11 @@ use std::{path::Path, pin::pin, sync::Arc}; use anyhow::{Context as _, Result, bail}; use arc_swap::ArcSwap; -use futures::{StreamExt as _, never::Never}; +use futures::StreamExt as _; use indexer_core::{IndexerCore, config::IndexerConfig}; -use indexer_service_protocol::{Account, AccountId, Block, BlockId, HashType, Transaction}; +use indexer_service_protocol::{ + Account, AccountId, Block, BlockId, HashType, IndexerStatus, Transaction, +}; use jsonrpsee::{ SubscriptionSink, core::{Serialize, SubscriptionResult, async_trait}, @@ -12,6 +14,7 @@ use jsonrpsee::{ }; use log::{debug, error, info, warn}; use tokio::sync::mpsc::UnboundedSender; +use tokio_util::sync::CancellationToken; pub struct IndexerService { subscription_service: SubscriptionService, @@ -19,9 +22,13 @@ pub struct IndexerService { } impl IndexerService { - pub fn new(config: IndexerConfig, storage_dir: &Path) -> Result { - let indexer = IndexerCore::new(config, storage_dir)?; - let subscription_service = SubscriptionService::spawn_new(indexer.clone()); + pub async fn new( + config: IndexerConfig, + storage_dir: &Path, + shutdown: CancellationToken, + ) -> Result { + let indexer = IndexerCore::new(config, storage_dir).await?; + let subscription_service = SubscriptionService::spawn_new(indexer.clone(), shutdown); Ok(Self { subscription_service, @@ -149,6 +156,10 @@ impl indexer_service_rpc::RpcServer for IndexerService { Ok(tx_res) } + async fn get_status(&self) -> Result { + Ok(self.indexer.status().into()) + } + async fn healthcheck(&self) -> Result<(), ErrorObjectOwned> { // Checking, that indexer can calculate last state let _ = self @@ -164,15 +175,21 @@ impl indexer_service_rpc::RpcServer for IndexerService { struct SubscriptionService { parts: ArcSwap, indexer: IndexerCore, + /// Cancellation token that is used to signal the subscription service to shut down. + /// + /// NOTE: This will auto-cancel on `Drop`, so if your token is shared with other parts + /// use [`CancellationToken::child_token()`] instead. + shutdown: CancellationToken, } impl SubscriptionService { - pub fn spawn_new(indexer: IndexerCore) -> Self { - let parts = Self::spawn_respond_subscribers_loop(indexer.clone()); + pub fn spawn_new(indexer: IndexerCore, shutdown: CancellationToken) -> Self { + let parts = Self::spawn_respond_subscribers_loop(indexer.clone(), shutdown.clone()); Self { parts: ArcSwap::new(Arc::new(parts)), indexer, + shutdown, } } @@ -184,14 +201,18 @@ impl SubscriptionService { ); // Respawn the subscription service loop if it has finished (either with error or panic) - if guard.handle.is_finished() { + if guard.handle.is_finished() && !self.shutdown.is_cancelled() { drop(guard); - let new_parts = Self::spawn_respond_subscribers_loop(self.indexer.clone()); + let new_parts = Self::spawn_respond_subscribers_loop( + self.indexer.clone(), + self.shutdown.clone(), + ); let old_handle_and_sender = self.parts.swap(Arc::new(new_parts)); let old_parts = Arc::into_inner(old_handle_and_sender) .expect("There should be no other references to the old handle and sender"); match old_parts.handle.await { + Ok(Ok(())) => {} Ok(Err(err)) => { error!( "Subscription service loop has unexpectedly finished with error: {err:#}" @@ -209,7 +230,10 @@ impl SubscriptionService { Ok(()) } - fn spawn_respond_subscribers_loop(indexer: IndexerCore) -> SubscriptionLoopParts { + fn spawn_respond_subscribers_loop( + indexer: IndexerCore, + shutdown: CancellationToken, + ) -> SubscriptionLoopParts { let (new_subscription_sender, mut sub_receiver) = tokio::sync::mpsc::unbounded_channel::>(); @@ -225,6 +249,10 @@ impl SubscriptionService { )] loop { tokio::select! { + () = shutdown.cancelled() => { + info!("Shutdown requested; stopping block ingestion"); + return Ok(()); + } sub = sub_receiver.recv() => { let Some(subscription) = sub else { bail!("Subscription receiver closed unexpectedly"); @@ -253,10 +281,11 @@ impl SubscriptionService { } } }; - let res: anyhow::Result = run_loop.await; - let Err(err) = res; - error!("Subscription service loop has unexpectedly finished with error: {err:#?}"); - Err(err) + let res: anyhow::Result<()> = run_loop.await; + if let Err(err) = &res { + error!("Subscription service loop has unexpectedly finished with error: {err:#?}"); + } + res }); SubscriptionLoopParts { handle, @@ -267,12 +296,13 @@ impl SubscriptionService { impl Drop for SubscriptionService { fn drop(&mut self) { + self.shutdown.cancel(); self.parts.load().handle.abort(); } } struct SubscriptionLoopParts { - handle: tokio::task::JoinHandle>, + handle: tokio::task::JoinHandle>, new_subscription_sender: UnboundedSender>, } diff --git a/lez/programs/bridge_lock/core/Cargo.toml b/lez/programs/bridge_lock/core/Cargo.toml index 4dadc075..190fc4f2 100644 --- a/lez/programs/bridge_lock/core/Cargo.toml +++ b/lez/programs/bridge_lock/core/Cargo.toml @@ -10,4 +10,3 @@ workspace = true [dependencies] lee_core.workspace = true serde = { workspace = true, features = ["alloc"] } - diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index 6cd1969f..cd89f223 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -41,6 +41,7 @@ url.workspace = true num-bigint.workspace = true risc0-zkvm.workspace = true futures.workspace = true +itertools.workspace = true [features] default = [] diff --git a/lez/sequencer/core/src/block_store.rs b/lez/sequencer/core/src/block_store.rs index 0db05d74..77b57661 100644 --- a/lez/sequencer/core/src/block_store.rs +++ b/lez/sequencer/core/src/block_store.rs @@ -7,18 +7,19 @@ use common::{ transaction::LeeTransaction, }; use lee::V03State; +use lee_core::BlockId; use log::info; use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint; -pub use storage::DbResult; use storage::sequencer::{ RocksDBIO, sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey}, }; +pub use storage::{DbResult, sequencer::DbDump}; pub struct SequencerStore { dbio: Arc, // TODO: Consider adding the hashmap to the database for faster recovery. - tx_hash_to_block_map: HashMap, + tx_hash_to_block_map: HashMap, genesis_id: u64, signing_key: lee::PrivateKey, } @@ -96,7 +97,7 @@ impl SequencerStore { /// Returns the transaction corresponding to the given hash, if it exists in the blockchain. #[must_use] - pub fn get_transaction_by_hash(&self, hash: HashType) -> Option { + pub fn get_transaction_by_hash(&self, hash: HashType) -> Option<(LeeTransaction, BlockId)> { let block_id = *self.tx_hash_to_block_map.get(&hash)?; let block = self .get_block_at_id(block_id) @@ -105,7 +106,7 @@ impl SequencerStore { .expect("Block should be present since the hash is in the map"); for transaction in block.body.transactions { if transaction.hash() == hash { - return Some(transaction); + return Some((transaction, block_id)); } } panic!( @@ -149,6 +150,28 @@ impl SequencerStore { self.dbio.get_lee_state() } + /// Remove the persisted zone-sdk checkpoint so the next startup is treated as a fresh start. + pub fn delete_zone_checkpoint(&self) -> DbResult<()> { + self.dbio.delete_zone_sdk_checkpoint_bytes() + } + + /// Reset every stored block to `Pending` so the next fresh start republishes the whole chain. + pub fn reset_all_blocks_to_pending(&self) -> DbResult<()> { + self.dbio.reset_all_blocks_to_pending() + } + + /// Single-blob [`DbDump`] of the whole store; restore with [`Self::restore_db_from_dump`]. + pub fn dump(&self) -> DbResult { + self.dbio.dump_all() + } + + /// Create a fresh rocksdb at `location` from `dump`, closing it before returning so a sequencer + /// can open it normally afterwards. + pub fn restore_db_from_dump(location: &Path, dump: &DbDump) -> DbResult<()> { + RocksDBIO::restore_from_dump(location, dump)?; + Ok(()) + } + pub fn get_zone_checkpoint(&self) -> Result> { let Some(bytes) = self.dbio.get_zone_sdk_checkpoint_bytes()? else { return Ok(None); @@ -181,8 +204,6 @@ pub(crate) fn block_to_transactions_map(block: &Block) -> HashMap #[cfg(test)] mod tests { - #![expect(clippy::shadow_unrelated, reason = "We don't care about it in tests")] - use common::{block::HashableBlockData, test_utils::sequencer_sign_key_for_testing}; use tempfile::tempdir; @@ -224,8 +245,8 @@ mod tests { .update(&block, &[], vec![], &dummy_state) .unwrap(); // Try again - let retrieved_tx = node_store.get_transaction_by_hash(tx.hash()); - assert_eq!(Some(tx), retrieved_tx); + let output = node_store.get_transaction_by_hash(tx.hash()); + assert_eq!(Some((tx, 1)), output); } #[test] @@ -383,7 +404,7 @@ mod tests { // Re-open the store and verify that the transaction is still retrievable (which means it // was cached correctly) let node_store = SequencerStore::open_db(path, signing_key).unwrap(); - let retrieved_tx = node_store.get_transaction_by_hash(tx.hash()); - assert_eq!(Some(tx), retrieved_tx); + let output = node_store.get_transaction_by_hash(tx.hash()); + assert_eq!(Some((tx, 1)), output); } } diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 995f361e..3a96e6d5 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -8,6 +8,7 @@ use common::{ transaction::{LeeTransaction, clock_invocation}, }; use config::{GenesisAction, SequencerConfig}; +use itertools::Itertools as _; use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::GENESIS_BLOCK_ID; use log::{error, info, warn}; @@ -125,7 +126,7 @@ impl SequencerCore { load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); - let (store, state, genesis_block) = Self::open_or_create_store(&config); + let (store, state, _genesis_block) = Self::open_or_create_store(&config); let latest_block_meta = store .latest_block_meta() @@ -152,15 +153,33 @@ impl SequencerCore { .await .expect("Failed to initialize Block Publisher"); - // On a truly fresh start (no checkpoint persisted yet), publish the - // genesis block so the indexer can find the channel start. After the - // first publish, zone-sdk's checkpoint persistence covers further - // restarts. + // Fresh start (no checkpoint): republish all pending blocks if is_fresh_start { - block_publisher - .publish_block(&genesis_block, vec![]) - .await - .expect("Failed to publish genesis block"); + let mut pending_blocks = store + .get_all_blocks() + .filter_ok(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) + .collect::, _>>() + .expect("Failed to read blocks from store while republishing on fresh start"); + pending_blocks.sort_unstable_by_key(|block| block.header.block_id); + + assert!( + pending_blocks + .first() + .is_none_or(|block| block.header.block_id == GENESIS_BLOCK_ID), + "First pending block on fresh start should be the genesis block" + ); + + for block in &pending_blocks { + block_publisher + .publish_block(block, vec![]) + .await + .unwrap_or_else(|err| { + panic!( + "Failed to publish block {} on fresh start: {err:#}", + block.header.block_id + ) + }); + } } // Cross-zone messaging: start a watcher per configured peer. The inbox diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 7e2bcdc5..05e2ec4c 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -10,15 +10,15 @@ use common::{ }; use key_protocol::key_management::KeyChain; use lee::{ - Account, AccountId, Data, EphemeralPublicKey, PrivacyPreservingTransaction, PrivateKey, - PublicKey, PublicTransaction, SharedSecretKey, V03State, + Account, AccountId, Data, PrivacyPreservingTransaction, PrivateKey, PublicKey, + PublicTransaction, V03State, error::LeeError, execute_and_prove, privacy_preserving_transaction::{Message, circuit::ProgramWithDependencies}, program::Program, }; use lee_core::{ - Commitment, EncryptedAccountData, InputAccountIdentity, Nullifier, + Commitment, InputAccountIdentity, Nullifier, account::{AccountWithMetadata, Nonce}, program::PdaSeed, }; @@ -803,8 +803,11 @@ async fn block_production_aborts_when_clock_account_data_is_corrupted() { #[test] fn private_bridge_withdraw_invocation_is_dropped() { let sender_keys = KeyChain::new_os_random(); - let sender_account_id = - AccountId::for_regular_private_account(&sender_keys.nullifier_public_key, 0); + let sender_account_id = AccountId::for_regular_private_account( + &sender_keys.nullifier_public_key, + &sender_keys.viewing_public_key, + 0, + ); let sender_private_account = Account { program_owner: programs::authenticated_transfer().id(), balance: 100, @@ -825,7 +828,11 @@ fn private_bridge_withdraw_invocation_is_dropped() { let sender_pre = AccountWithMetadata::new( sender_private_account, true, - (&sender_keys.nullifier_public_key, 0), + ( + &sender_keys.nullifier_public_key, + &sender_keys.viewing_public_key, + 0, + ), ); let bridge_pre = AccountWithMetadata::new( state.get_account_by_id(bridge_account_id), @@ -833,8 +840,6 @@ fn private_bridge_withdraw_invocation_is_dropped() { bridge_account_id, ); - let shared_secret = SharedSecretKey::encapsulate(&sender_keys.viewing_public_key).0; - let instruction = Program::serialize_instruction(bridge_core::Instruction::Withdraw { amount: 1, bedrock_account_pk: [0; 32], @@ -855,12 +860,8 @@ fn private_bridge_withdraw_invocation_is_dropped() { instruction, vec![ InputAccountIdentity::PrivateAuthorizedUpdate { - epk: EphemeralPublicKey(vec![12_u8; 1088]), - view_tag: EncryptedAccountData::compute_view_tag( - &sender_keys.nullifier_public_key, - &sender_keys.viewing_public_key, - ), - ssk: shared_secret, + vpk: sender_keys.viewing_public_key.clone(), + random_seed: [0; 32], nsk: sender_keys.private_key_holder.nullifier_secret_key, membership_proof: state .get_proof_for_commitment(&sender_commitment) diff --git a/lez/sequencer/service/rpc/src/lib.rs b/lez/sequencer/service/rpc/src/lib.rs index 914232c0..f7aa8b56 100644 --- a/lez/sequencer/service/rpc/src/lib.rs +++ b/lez/sequencer/service/rpc/src/lib.rs @@ -68,7 +68,7 @@ pub trait Rpc { async fn get_transaction( &self, tx_hash: HashType, - ) -> Result, ErrorObjectOwned>; + ) -> Result, ErrorObjectOwned>; #[method(name = "getAccountsNonces")] async fn get_accounts_nonces( diff --git a/lez/sequencer/service/src/main.rs b/lez/sequencer/service/src/main.rs index e78ad502..8b577bb8 100644 --- a/lez/sequencer/service/src/main.rs +++ b/lez/sequencer/service/src/main.rs @@ -24,6 +24,8 @@ async fn main() -> Result<()> { let Args { config_path, port } = Args::parse(); + // TODO: handle this cancellation token more gracefully within Sequencer service + // similar to how we do in Indexer let cancellation_token = listen_for_shutdown_signal(); let config = sequencer_service::SequencerConfig::from_path(&config_path)?; diff --git a/lez/sequencer/service/src/service.rs b/lez/sequencer/service/src/service.rs index 4ccac9b4..60be5fdd 100644 --- a/lez/sequencer/service/src/service.rs +++ b/lez/sequencer/service/src/service.rs @@ -145,7 +145,7 @@ impl sequencer_service_rpc::RpcServer async fn get_transaction( &self, tx_hash: HashType, - ) -> Result, ErrorObjectOwned> { + ) -> Result, ErrorObjectOwned> { let sequencer = self.sequencer.lock().await; Ok(sequencer.block_store().get_transaction_by_hash(tx_hash)) } diff --git a/lez/storage/Cargo.toml b/lez/storage/Cargo.toml index 8767b525..18c58e11 100644 --- a/lez/storage/Cargo.toml +++ b/lez/storage/Cargo.toml @@ -13,8 +13,10 @@ lee.workspace = true thiserror.workspace = true borsh.workspace = true +log.workspace = true rocksdb.workspace = true tempfile.workspace = true +zstd.workspace = true [dev-dependencies] programs.workspace = true diff --git a/lez/storage/src/error.rs b/lez/storage/src/error.rs index 3056e09b..069a95d8 100644 --- a/lez/storage/src/error.rs +++ b/lez/storage/src/error.rs @@ -12,6 +12,12 @@ pub enum DbError { error: borsh::io::Error, additional_info: Option, }, + #[error("Compression error: {}", additional_info.as_deref().unwrap_or("No additional info"))] + CompressionError { + #[source] + error: std::io::Error, + additional_info: Option, + }, #[error("Logic Error: {additional_info}")] DbInteractionError { additional_info: String }, } @@ -33,6 +39,14 @@ impl DbError { } } + #[must_use] + pub const fn compression_error(err: std::io::Error, message: Option) -> Self { + Self::CompressionError { + error: err, + additional_info: message, + } + } + #[must_use] pub const fn db_interaction_error(message: String) -> Self { Self::DbInteractionError { diff --git a/lez/storage/src/indexer/indexer_cells.rs b/lez/storage/src/indexer/indexer_cells.rs index b19a5510..ef104f1a 100644 --- a/lez/storage/src/indexer/indexer_cells.rs +++ b/lez/storage/src/indexer/indexer_cells.rs @@ -7,9 +7,9 @@ use crate::{ error::DbError, indexer::{ ACC_NUM_CELL_NAME, BLOCK_HASH_CELL_NAME, BREAKPOINT_CELL_NAME, CF_ACC_META, - CF_BREAKPOINT_NAME, CF_HASH_TO_ID, CF_TX_TO_ID, DB_META_LAST_BREAKPOINT_ID, - DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DB_META_ZONE_SDK_INDEXER_CURSOR_KEY, - TX_HASH_CELL_NAME, + CF_BREAKPOINT_NAME, CF_HASH_TO_ID, CF_TX_TO_ID, + DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY, DB_META_STALL_REASON_KEY, + DB_META_TIP_SLOT_KEY, DB_META_ZONE_SDK_INDEXER_CURSOR_KEY, TX_HASH_CELL_NAME, }, }; @@ -36,29 +36,6 @@ impl SimpleWritableCell for LastObservedL1LibHeaderCell { } } -#[derive(Debug, BorshSerialize, BorshDeserialize)] -pub struct LastBreakpointIdCell(pub u64); - -impl SimpleStorableCell for LastBreakpointIdCell { - type KeyParams = (); - - const CELL_NAME: &'static str = DB_META_LAST_BREAKPOINT_ID; - const CF_NAME: &'static str = CF_META_NAME; -} - -impl SimpleReadableCell for LastBreakpointIdCell {} - -impl SimpleWritableCell for LastBreakpointIdCell { - fn value_constructor(&self) -> DbResult> { - borsh::to_vec(&self).map_err(|err| { - DbError::borsh_cast_message( - err, - Some("Failed to serialize last breakpoint id".to_owned()), - ) - }) - } -} - #[derive(BorshDeserialize)] pub struct BreakpointCellOwned(pub V03State); @@ -212,6 +189,27 @@ impl SimpleWritableCell for AccNumTxCell { } } +/// The L1 inscription slot of the tip block, written atomically with the tip. +#[derive(Debug, BorshSerialize, BorshDeserialize)] +pub struct TipSlotCell(pub u64); + +impl SimpleStorableCell for TipSlotCell { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_META_TIP_SLOT_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleReadableCell for TipSlotCell {} + +impl SimpleWritableCell for TipSlotCell { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize tip slot".to_owned())) + }) + } +} + /// Opaque bytes for the zone-sdk indexer cursor `Option<(MsgId, Slot)>`. /// The caller serializes via `serde_json` (neither type derives borsh). #[derive(BorshDeserialize)] @@ -247,6 +245,40 @@ impl SimpleWritableCell for ZoneSdkIndexerCursorCellRef<'_> { } } +/// Opaque JSON bytes for the indexer's persisted `Option`. +#[derive(BorshDeserialize)] +pub struct StallReasonCellOwned(pub Vec); + +impl SimpleStorableCell for StallReasonCellOwned { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_META_STALL_REASON_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleReadableCell for StallReasonCellOwned {} + +#[derive(BorshSerialize)] +pub struct StallReasonCellRef<'bytes>(pub &'bytes [u8]); + +impl SimpleStorableCell for StallReasonCellRef<'_> { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_META_STALL_REASON_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleWritableCell for StallReasonCellRef<'_> { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message( + err, + Some("Failed to serialize stall reason cell".to_owned()), + ) + }) + } +} + #[cfg(test)] mod uniform_tests { use crate::{ diff --git a/lez/storage/src/indexer/mod.rs b/lez/storage/src/indexer/mod.rs index 87d586fb..0955ef26 100644 --- a/lez/storage/src/indexer/mod.rs +++ b/lez/storage/src/indexer/mod.rs @@ -5,6 +5,7 @@ use common::{ transaction::{LeeTransaction, clock_invocation}, }; use lee::{GENESIS_BLOCK_ID, V03State}; +use log::warn; use rocksdb::{ BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, }; @@ -20,10 +21,12 @@ pub mod write_non_atomic; /// Key base for storing metainformation about id of last observed L1 lib header in db. pub const DB_META_LAST_OBSERVED_L1_LIB_HEADER_ID_IN_DB_KEY: &str = "last_observed_l1_lib_header_in_db"; -/// Key base for storing metainformation about the last breakpoint. -pub const DB_META_LAST_BREAKPOINT_ID: &str = "last_breakpoint_id"; /// Key base for storing the zone-sdk indexer cursor (opaque bytes). pub const DB_META_ZONE_SDK_INDEXER_CURSOR_KEY: &str = "zone_sdk_indexer_cursor"; +/// Key base for storing the persisted `Option` diagnostic record (opaque JSON bytes). +pub const DB_META_STALL_REASON_KEY: &str = "stall_reason"; +/// Key base for storing the L1 inscription slot of the tip block. +pub const DB_META_TIP_SLOT_KEY: &str = "tip_slot"; /// Cell name for a breakpoint. pub const BREAKPOINT_CELL_NAME: &str = "breakpoint"; @@ -84,9 +87,10 @@ impl RocksDBIO { let dbio = Self { db }; - // First breakpoint setup - dbio.put_breakpoint(0, initial_state)?; - dbio.put_meta_last_breakpoint_id(0)?; + // Seed the genesis snapshot once; reopening must not clobber it. + if dbio.get_breakpoint_opt(0)?.is_none() { + dbio.put_breakpoint(0, initial_state)?; + } Ok(dbio) } @@ -152,8 +156,29 @@ impl RocksDBIO { )); } - let br_id = closest_breakpoint_id(block_id); - let mut state = self.get_breakpoint(br_id)?; + // walk down to the nearest snapshot that exists + let target = closest_breakpoint_id(block_id); + let mut br_id = target; + let mut state = loop { + match self.get_breakpoint_opt(br_id)? { + Some(state) => break state, + None if br_id == 0 => { + return Err(DbError::db_interaction_error( + "Breakpoint 0 is missing".to_owned(), + )); + } + None => { + br_id = br_id + .checked_sub(1) + .expect("breakpoint_id > 0 checked above"); + } + } + }; + if br_id < target { + warn!( + "Breakpoint {target} missing; replaying from breakpoint {br_id} for block {block_id}" + ); + } let start = u64::from(BREAKPOINT_INTERVAL) .checked_mul(br_id) @@ -211,20 +236,7 @@ fn apply_block_transactions(mut block: Block, state: &mut V03State) -> DbResult< })?; } else { transaction - .transaction_stateless_check() - .map_err(|err| { - DbError::db_interaction_error(format!( - "transaction pre check failed with err {err:?}" - )) - })? - // FIXME: HOT FIX (testnet v0.2): does not check for system account updates due to - // sequencer-generated deposit tx'es; - // CHANGE ME back to `execute_check_on_state` when the indexer can authenticate deposit transactions - .execute_without_system_accounts_check_on_state( - state, - block.header.block_id, - block.header.timestamp, - ) + .execute_on_state(state, block.header.block_id, block.header.timestamp) .map_err(|err| { DbError::db_interaction_error(format!( "transaction execution failed with err {err:?}" diff --git a/lez/storage/src/indexer/read_once.rs b/lez/storage/src/indexer/read_once.rs index 6e79adc4..3fbf6026 100644 --- a/lez/storage/src/indexer/read_once.rs +++ b/lez/storage/src/indexer/read_once.rs @@ -3,8 +3,8 @@ use crate::{ DBIO as _, cells::shared_cells::{BlockCell, FirstBlockCell, FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ - AccNumTxCell, BlockHashToBlockIdMapCell, BreakpointCellOwned, LastBreakpointIdCell, - LastObservedL1LibHeaderCell, TxHashToBlockIdMapCell, ZoneSdkIndexerCursorCellOwned, + AccNumTxCell, BlockHashToBlockIdMapCell, BreakpointCellOwned, LastObservedL1LibHeaderCell, + StallReasonCellOwned, TipSlotCell, TxHashToBlockIdMapCell, ZoneSdkIndexerCursorCellOwned, }, }; @@ -31,8 +31,8 @@ impl RocksDBIO { Ok(self.get_opt::(())?.is_some()) } - pub fn get_meta_last_breakpoint_id(&self) -> DbResult> { - self.get_opt::(()) + pub fn get_meta_tip_slot_in_db(&self) -> DbResult> { + self.get_opt::(()) .map(|opt| opt.map(|cell| cell.0)) } @@ -49,6 +49,11 @@ impl RocksDBIO { self.get::(br_id).map(|cell| cell.0) } + pub fn get_breakpoint_opt(&self, br_id: u64) -> DbResult> { + self.get_opt::(br_id) + .map(|opt| opt.map(|cell| cell.0)) + } + // Mappings pub fn get_block_id_by_hash(&self, hash: [u8; 32]) -> DbResult> { @@ -73,4 +78,8 @@ impl RocksDBIO { .get_opt::(())? .map(|cell| cell.0)) } + + pub fn get_stall_reason_bytes(&self) -> DbResult>> { + Ok(self.get_opt::(())?.map(|cell| cell.0)) + } } diff --git a/lez/storage/src/indexer/tests.rs b/lez/storage/src/indexer/tests.rs index 44df23d5..d87aaf1c 100644 --- a/lez/storage/src/indexer/tests.rs +++ b/lez/storage/src/indexer/tests.rs @@ -49,16 +49,16 @@ fn initial_state() -> lee::V03State { #[test] fn start_db() { + let initial_state = initial_state(); let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap(); let first_id = dbio.get_meta_first_block_id_in_db().unwrap(); let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); let last_observed_l1_header = dbio.get_meta_last_observed_l1_lib_header_in_db().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); let last_block = dbio.get_block(1).unwrap(); let breakpoint = dbio.get_breakpoint(0).unwrap(); let final_state = dbio.final_state().unwrap(); @@ -67,7 +67,6 @@ fn start_db() { assert_eq!(first_id, None); assert_eq!(last_observed_l1_header, None); assert!(!is_first_set); - assert_eq!(last_br_id, Some(0)); // TODO: Will be None after we remove hardcoded testnet state assert!(last_block.is_none()); assert_eq!( breakpoint.get_account_by_id(acc1()), @@ -81,13 +80,15 @@ fn start_db() { #[test] fn one_block_insertion() { + let initial_state = initial_state(); let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state).unwrap(); let genesis_block = genesis_block(); - dbio.put_block(&genesis_block, [0; 32]).unwrap(); + dbio.put_block(&genesis_block, [0; 32], 0, &initial_state) + .unwrap(); let prev_hash = genesis_block.header.hash; let from = acc1(); @@ -98,7 +99,7 @@ fn one_block_insertion() { common::test_utils::create_transaction_native_token_transfer(from, 0, to, 1, &sign_key); let block = produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(&block, [1; 32]).unwrap(); + dbio.put_block(&block, [1; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let first_id = dbio.get_meta_first_block_id_in_db().unwrap(); @@ -107,7 +108,6 @@ fn one_block_insertion() { .unwrap() .unwrap(); let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); let breakpoint = dbio.get_breakpoint(0).unwrap(); let final_state = dbio.final_state().unwrap(); @@ -116,7 +116,6 @@ fn one_block_insertion() { assert_eq!(first_id, Some(1)); assert_eq!(last_observed_l1_header, [1; 32]); assert!(is_first_set); - assert_eq!(last_br_id, Some(0)); assert_eq!(last_block.header.hash, block.header.hash); assert_eq!( breakpoint.get_account_by_id(acc1()).balance @@ -131,17 +130,44 @@ fn one_block_insertion() { } #[test] -fn new_breakpoint() { +fn put_block_records_tip_inscription_slot() { + let initial_state = initial_state(); let temp_dir = tempdir().unwrap(); - let temdir_path = temp_dir.path(); + let dbio = RocksDBIO::open_or_create(temp_dir.path(), &initial_state).unwrap(); - let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state()).unwrap(); + assert_eq!(dbio.get_meta_tip_slot_in_db().unwrap(), None); + + let genesis_block = genesis_block(); + dbio.put_block(&genesis_block, [0; 32], 1_000, &initial_state) + .unwrap(); + assert_eq!(dbio.get_meta_tip_slot_in_db().unwrap(), Some(1_000)); + + let block = produce_dummy_block(2, Some(genesis_block.header.hash), vec![]); + dbio.put_block(&block, [1; 32], 1_005, &initial_state) + .unwrap(); + assert_eq!(dbio.get_meta_tip_slot_in_db().unwrap(), Some(1_005)); + + // Re-inserting a block at/below the tip must not move the tip slot. + dbio.put_block(&genesis_block, [0; 32], 1_010, &initial_state) + .unwrap(); + assert_eq!(dbio.get_meta_tip_slot_in_db().unwrap(), Some(1_005)); +} + +#[test] +fn put_block_stores_breakpoint_in_same_batch() { + let initial_state = initial_state(); + let temp_dir = tempdir().unwrap(); + let dbio = RocksDBIO::open_or_create(temp_dir.path(), &initial_state).unwrap(); let from = acc1(); let to = acc2(); let sign_key = acc1_sign_key(); - for i in 1..=BREAKPOINT_INTERVAL + 1 { + // Chain blocks 1..=BREAKPOINT_INTERVAL. The snapshot is scheduled internally + // by put_block at the boundary block; every call passes the same recognizable + // marker state (the initial one), proving it's stored verbatim rather than + // recomputed. + for i in 1..=BREAKPOINT_INTERVAL { let prev_hash = dbio.get_meta_last_block_id_in_db().unwrap().map(|last_id| { let last_block = dbio.get_block(last_id).unwrap().unwrap(); last_block.header.hash @@ -155,51 +181,70 @@ fn new_breakpoint() { &sign_key, ); let block = produce_dummy_block(i.into(), prev_hash, vec![transfer_tx]); - dbio.put_block(&block, [i; 32]).unwrap(); + + dbio.put_block(&block, [i; 32], 0, &initial_state).unwrap(); } - let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); - let first_id = dbio.get_meta_first_block_id_in_db().unwrap(); - let is_first_set = dbio.get_meta_is_first_block_set().unwrap(); - let last_br_id = dbio.get_meta_last_breakpoint_id().unwrap(); - let last_block = dbio.get_block(last_id).unwrap().unwrap(); - let prev_breakpoint = dbio.get_breakpoint(0).unwrap(); - let breakpoint = dbio.get_breakpoint(1).unwrap(); - let final_state = dbio.final_state().unwrap(); + let bp1 = dbio.get_breakpoint(1).unwrap(); + assert_eq!(bp1.get_account_by_id(acc1()).balance, 10000); + assert_eq!(bp1.get_account_by_id(acc2()).balance, 20000); + // Only the boundary block schedules a write: breakpoint 0 must be the only other one. + assert_eq!( + dbio.get_breakpoint(0) + .unwrap() + .get_account_by_id(acc1()) + .balance, + 10000 + ); +} - assert_eq!(last_id, 101); - assert_eq!(first_id, Some(1)); - assert!(is_first_set); - assert_eq!(last_br_id, Some(1)); - assert_ne!(last_block.header.hash, genesis_block().header.hash); +#[test] +fn state_replay_falls_back_over_missing_breakpoints() { + let initial_state = initial_state(); + let temp_dir = tempdir().unwrap(); + let dbio = RocksDBIO::open_or_create(temp_dir.path(), &initial_state).unwrap(); + + let from = acc1(); + let to = acc2(); + let sign_key = acc1_sign_key(); + + for i in 1..=u64::from(BREAKPOINT_INTERVAL) + 1 { + let prev_hash = dbio.get_meta_last_block_id_in_db().unwrap().map(|last_id| { + let last_block = dbio.get_block(last_id).unwrap().unwrap(); + last_block.header.hash + }); + let transfer_tx = common::test_utils::create_transaction_native_token_transfer( + from, + (i - 1).into(), + to, + 1, + &sign_key, + ); + let block = produce_dummy_block(i, prev_hash, vec![transfer_tx]); + dbio.put_block(&block, [0; 32], 0, &initial_state).unwrap(); + } + + // Simulate a store whose boundary snapshot was lost (#605). + dbio.delete_breakpoint(1).unwrap(); + assert!(dbio.get_breakpoint_opt(1).unwrap().is_none()); + let final_state = dbio.final_state().unwrap(); assert_eq!( - prev_breakpoint.get_account_by_id(acc1()).balance - - final_state.get_account_by_id(acc1()).balance, - 101 + 10000 - final_state.get_account_by_id(acc1()).balance, + u128::from(BREAKPOINT_INTERVAL) + 1 ); assert_eq!( - final_state.get_account_by_id(acc2()).balance - - prev_breakpoint.get_account_by_id(acc2()).balance, - 101 - ); - assert_eq!( - breakpoint.get_account_by_id(acc1()).balance - - final_state.get_account_by_id(acc1()).balance, - 1 - ); - assert_eq!( - final_state.get_account_by_id(acc2()).balance - - breakpoint.get_account_by_id(acc2()).balance, - 1 + final_state.get_account_by_id(acc2()).balance - 20000, + u128::from(BREAKPOINT_INTERVAL) + 1 ); } #[test] fn simple_maps() { + let initial_state = initial_state(); let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state).unwrap(); let from = acc1(); let to = acc2(); @@ -211,7 +256,7 @@ fn simple_maps() { let control_hash1 = block.header.hash; - dbio.put_block(&block, [1; 32]).unwrap(); + dbio.put_block(&block, [1; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -223,7 +268,7 @@ fn simple_maps() { let control_hash2 = block.header.hash; - dbio.put_block(&block, [2; 32]).unwrap(); + dbio.put_block(&block, [2; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -235,7 +280,7 @@ fn simple_maps() { let control_tx_hash1 = transfer_tx.hash(); let block = produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(&block, [3; 32]).unwrap(); + dbio.put_block(&block, [3; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -247,7 +292,7 @@ fn simple_maps() { let control_tx_hash2 = transfer_tx.hash(); let block = produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(&block, [4; 32]).unwrap(); + dbio.put_block(&block, [4; 32], 0, &initial_state).unwrap(); let control_block_id1 = dbio.get_block_id_by_hash(control_hash1.0).unwrap().unwrap(); let control_block_id2 = dbio.get_block_id_by_hash(control_hash2.0).unwrap().unwrap(); @@ -268,12 +313,13 @@ fn simple_maps() { #[test] fn block_batch() { + let initial_state = initial_state(); let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); let mut block_res = vec![]; - let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state).unwrap(); let from = acc1(); let to = acc2(); @@ -284,7 +330,7 @@ fn block_batch() { let block = produce_dummy_block(1, None, vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(&block, [1; 32]).unwrap(); + dbio.put_block(&block, [1; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -295,7 +341,7 @@ fn block_batch() { let block = produce_dummy_block(2, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(&block, [2; 32]).unwrap(); + dbio.put_block(&block, [2; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -306,7 +352,7 @@ fn block_batch() { let block = produce_dummy_block(3, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(&block, [3; 32]).unwrap(); + dbio.put_block(&block, [3; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -317,7 +363,7 @@ fn block_batch() { let block = produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); block_res.push(block.clone()); - dbio.put_block(&block, [4; 32]).unwrap(); + dbio.put_block(&block, [4; 32], 0, &initial_state).unwrap(); let block_hashes_mem: Vec<[u8; 32]> = block_res.into_iter().map(|bl| bl.header.hash.0).collect(); @@ -356,10 +402,11 @@ fn block_batch() { #[test] fn account_map() { + let initial_state = initial_state(); let temp_dir = tempdir().unwrap(); let temdir_path = temp_dir.path(); - let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state()).unwrap(); + let dbio = RocksDBIO::open_or_create(temdir_path, &initial_state).unwrap(); let from = acc1(); let to = acc2(); @@ -376,7 +423,7 @@ fn account_map() { let block = produce_dummy_block(1, None, vec![transfer_tx1, transfer_tx2]); - dbio.put_block(&block, [1; 32]).unwrap(); + dbio.put_block(&block, [1; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -391,7 +438,7 @@ fn account_map() { let block = produce_dummy_block(2, Some(prev_hash), vec![transfer_tx1, transfer_tx2]); - dbio.put_block(&block, [2; 32]).unwrap(); + dbio.put_block(&block, [2; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -406,7 +453,7 @@ fn account_map() { let block = produce_dummy_block(3, Some(prev_hash), vec![transfer_tx1, transfer_tx2]); - dbio.put_block(&block, [3; 32]).unwrap(); + dbio.put_block(&block, [3; 32], 0, &initial_state).unwrap(); let last_id = dbio.get_meta_last_block_id_in_db().unwrap().unwrap(); let last_block = dbio.get_block(last_id).unwrap().unwrap(); @@ -418,7 +465,7 @@ fn account_map() { let block = produce_dummy_block(4, Some(prev_hash), vec![transfer_tx]); - dbio.put_block(&block, [4; 32]).unwrap(); + dbio.put_block(&block, [4; 32], 0, &initial_state).unwrap(); let acc1_tx = dbio.get_acc_transactions(*acc1().value(), 0, 7).unwrap(); let acc1_tx_hashes: Vec<[u8; 32]> = acc1_tx.into_iter().map(|tx| tx.hash().0).collect(); @@ -431,3 +478,15 @@ fn account_map() { assert_eq!(acc1_tx_limited_hashes.as_slice(), &tx_hash_res[1..5]); } + +#[test] +fn reopen_preserves_seeded_breakpoint() { + let initial_state = initial_state(); + let temp_dir = tempdir().unwrap(); + { + let dbio = RocksDBIO::open_or_create(temp_dir.path(), &initial_state).unwrap(); + assert!(dbio.get_breakpoint_opt(0).unwrap().is_some()); + } // drop releases the RocksDB lock + let dbio = RocksDBIO::open_or_create(temp_dir.path(), &initial_state).unwrap(); + assert!(dbio.get_breakpoint_opt(0).unwrap().is_some()); +} diff --git a/lez/storage/src/indexer/write_atomic.rs b/lez/storage/src/indexer/write_atomic.rs index ec28f8ec..729bcc02 100644 --- a/lez/storage/src/indexer/write_atomic.rs +++ b/lez/storage/src/indexer/write_atomic.rs @@ -2,13 +2,13 @@ use std::collections::HashMap; use rocksdb::WriteBatch; -use super::{BREAKPOINT_INTERVAL, Block, DbError, DbResult, RocksDBIO}; +use super::{BREAKPOINT_INTERVAL, Block, DbError, DbResult, RocksDBIO, V03State}; use crate::{ DBIO as _, cells::shared_cells::{FirstBlockCell, FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ - AccNumTxCell, BlockHashToBlockIdMapCell, LastBreakpointIdCell, LastObservedL1LibHeaderCell, - TxHashToBlockIdMapCell, + AccNumTxCell, BlockHashToBlockIdMapCell, BreakpointCellRef, LastObservedL1LibHeaderCell, + TipSlotCell, TxHashToBlockIdMapCell, }, }; @@ -131,21 +131,28 @@ impl RocksDBIO { self.put_batch(&LastObservedL1LibHeaderCell(l1_lib_header), (), write_batch) } - pub fn put_meta_last_breakpoint_id_batch( + pub fn put_meta_tip_slot_in_db_batch( &self, - br_id: u64, + l1_slot: u64, write_batch: &mut WriteBatch, ) -> DbResult<()> { - self.put_batch(&LastBreakpointIdCell(br_id), (), write_batch) + self.put_batch(&TipSlotCell(l1_slot), (), write_batch) } pub fn put_meta_is_first_block_set_batch(&self, write_batch: &mut WriteBatch) -> DbResult<()> { self.put_batch(&FirstBlockSetCell(true), (), write_batch) } - // Block - - pub fn put_block(&self, block: &Block, l1_lib_header: [u8; 32]) -> DbResult<()> { + /// Put a block atomically (via [`WriteBatch`]) along with its L1 header, `Slot`, + /// and (at interval-boundary blocks) a snapshot of `post_state`, the block's + /// post-application state. + pub fn put_block( + &self, + block: &Block, + l1_lib_header: [u8; 32], + l1_slot: u64, + post_state: &V03State, + ) -> DbResult<()> { let cf_block = self.block_column(); let last_curr_block = self.get_meta_last_block_id_in_db()?.unwrap_or(0); let mut write_batch = WriteBatch::default(); @@ -163,6 +170,7 @@ impl RocksDBIO { if block.header.block_id > last_curr_block { self.put_meta_last_block_in_db_batch(block.header.block_id, &mut write_batch)?; self.put_meta_last_observed_l1_lib_header_in_db_batch(l1_lib_header, &mut write_batch)?; + self.put_meta_tip_slot_in_db_batch(l1_slot, &mut write_batch)?; } if last_curr_block == 0 { self.put_meta_first_block_in_db_batch(block, &mut write_batch)?; @@ -208,18 +216,23 @@ impl RocksDBIO { self.put_account_transactions_dependant(acc_id, &tx_hashes, &mut write_batch)?; } - self.db.write(write_batch).map_err(|rerr| { - DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_owned())) - })?; - if block .header .block_id .is_multiple_of(BREAKPOINT_INTERVAL.into()) { - self.put_next_breakpoint()?; + let br_id = block + .header + .block_id + .checked_div(BREAKPOINT_INTERVAL.into()) + .expect("Breakpoint interval is not zero"); + self.put_batch(&BreakpointCellRef(post_state), br_id, &mut write_batch)?; } + self.db.write(write_batch).map_err(|rerr| { + DbError::rocksdb_cast_message(rerr, Some("Failed to write batch".to_owned())) + })?; + Ok(()) } } diff --git a/lez/storage/src/indexer/write_non_atomic.rs b/lez/storage/src/indexer/write_non_atomic.rs index 7ddab1dd..a2c178ec 100644 --- a/lez/storage/src/indexer/write_non_atomic.rs +++ b/lez/storage/src/indexer/write_non_atomic.rs @@ -1,9 +1,11 @@ -use super::{BREAKPOINT_INTERVAL, DbError, DbResult, RocksDBIO, V03State}; +use super::{DbResult, RocksDBIO, V03State}; +#[cfg(test)] +use crate::error::DbError; use crate::{ DBIO as _, cells::shared_cells::{FirstBlockSetCell, LastBlockCell}, indexer::indexer_cells::{ - BreakpointCellRef, LastBreakpointIdCell, LastObservedL1LibHeaderCell, + BreakpointCellRef, LastObservedL1LibHeaderCell, StallReasonCellRef, ZoneSdkIndexerCursorCellRef, }, }; @@ -23,10 +25,6 @@ impl RocksDBIO { self.put(&LastObservedL1LibHeaderCell(l1_lib_header), ()) } - pub fn put_meta_last_breakpoint_id(&self, br_id: u64) -> DbResult<()> { - self.put(&LastBreakpointIdCell(br_id), ()) - } - pub fn put_meta_is_first_block_set(&self) -> DbResult<()> { self.put(&FirstBlockSetCell(true), ()) } @@ -35,32 +33,25 @@ impl RocksDBIO { self.put(&ZoneSdkIndexerCursorCellRef(bytes), ()) } + pub fn put_stall_reason_bytes(&self, bytes: &[u8]) -> DbResult<()> { + self.put(&StallReasonCellRef(bytes), ()) + } + // State pub fn put_breakpoint(&self, br_id: u64, breakpoint: &V03State) -> DbResult<()> { self.put(&BreakpointCellRef(breakpoint), br_id) } - pub fn put_next_breakpoint(&self) -> DbResult<()> { - let last_block = self.get_meta_last_block_id_in_db()?.unwrap_or(0); - let next_breakpoint_id = self - .get_meta_last_breakpoint_id()? - .unwrap_or(0) - .checked_add(1) - .expect("Breakpoint Id will be lesser than u64::MAX"); - let block_to_break_id = next_breakpoint_id - .checked_mul(u64::from(BREAKPOINT_INTERVAL)) - .expect("Reached maximum breakpoint id"); - - if block_to_break_id <= last_block { - let next_breakpoint = self.calculate_state_for_id(block_to_break_id)?; - - self.put_breakpoint(next_breakpoint_id, &next_breakpoint)?; - self.put_meta_last_breakpoint_id(next_breakpoint_id) - } else { - Err(DbError::db_interaction_error( - "Breakpoint not yet achieved".to_owned(), - )) - } + /// Deletes a breakpoint snapshot. Test-only fault injection for simulating + /// stores whose boundary snapshot was lost. + #[cfg(test)] + pub(crate) fn delete_breakpoint(&self, br_id: u64) -> DbResult<()> { + let key = borsh::to_vec(&br_id).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize breakpoint id".to_owned())) + })?; + self.db + .delete_cf(&self.breakpoint_column(), key) + .map_err(|rerr| DbError::rocksdb_cast_message(rerr, None)) } } diff --git a/lez/storage/src/lib.rs b/lez/storage/src/lib.rs index 2edb0ee3..8fd485d3 100644 --- a/lez/storage/src/lib.rs +++ b/lez/storage/src/lib.rs @@ -1,7 +1,7 @@ use rocksdb::{DBWithThreadMode, MultiThreaded, WriteBatch}; use crate::{ - cells::{SimpleReadableCell, SimpleWritableCell}, + cells::{SimpleReadableCell, SimpleStorableCell, SimpleWritableCell}, error::DbError, }; @@ -66,4 +66,17 @@ pub trait DBIO { ) -> DbResult<()> { cell.put_batch(self.db(), params, write_batch) } + + /// Delete a cell. Deleting an absent key is a no-op (rocksdb semantics). + fn del(&self, params: T::KeyParams) -> DbResult<()> { + let cf_ref = T::column_ref(self.db()); + self.db() + .delete_cf(&cf_ref, T::key_constructor(params)?) + .map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some(format!("Failed to delete {:?}", T::CELL_NAME)), + ) + }) + } } diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index 44068517..1a3a1886 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -1,12 +1,14 @@ use std::{path::Path, sync::Arc}; +use borsh::{BorshDeserialize, BorshSerialize}; use common::{ HashType, block::{BedrockStatus, Block, BlockMeta}, }; use lee::V03State; use rocksdb::{ - BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, MultiThreaded, Options, WriteBatch, + BoundColumnFamily, ColumnFamilyDescriptor, DBWithThreadMode, IteratorMode, MultiThreaded, + Options, WriteBatch, }; use crate::{ @@ -44,6 +46,47 @@ pub const DB_LEE_STATE_KEY: &str = "lee_state"; /// Name of state column family. pub const CF_LEE_STATE_NAME: &str = "cf_lee_state"; +/// A single key/value entry from a column family, used inside [`DbDump`]. +#[derive(BorshSerialize, BorshDeserialize)] +pub struct DbDumpEntry { + pub cf_name: String, + pub key: Vec, + pub value: Vec, +} + +/// Schema-agnostic single-blob snapshot of a store: every key/value pair across all column +/// families. Lets a prebuilt store ship as one committed file instead of a rocksdb directory. +#[derive(BorshSerialize, BorshDeserialize)] +pub struct DbDump { + pub entries: Vec, +} + +impl DbDump { + /// Serialize the dump to a zstd-compressed borsh blob. + pub fn to_bytes(&self) -> DbResult> { + /// zstd compression level for [`DbDump::to_bytes`]. Level 19 keeps the committed fixture + /// small without a meaningful decompression cost. + const DUMP_ZSTD_LEVEL: i32 = 19; + + let borsh = borsh::to_vec(self).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize DbDump".to_owned())) + })?; + zstd::encode_all(borsh.as_slice(), DUMP_ZSTD_LEVEL).map_err(|err| { + DbError::compression_error(err, Some("Failed to compress DbDump".to_owned())) + }) + } + + /// Deserialize a dump produced by [`Self::to_bytes`]. + pub fn from_bytes(bytes: &[u8]) -> DbResult { + let borsh = zstd::decode_all(bytes).map_err(|err| { + DbError::db_interaction_error(format!("Failed to decompress DbDump: {err}")) + })?; + borsh::from_slice(&borsh).map_err(|err| { + DbError::compression_error(err, Some("Failed to deserialize DbDump".to_owned())) + }) + } +} + pub struct RocksDBIO { pub db: DBWithThreadMode, } @@ -84,6 +127,71 @@ impl RocksDBIO { Ok(dbio) } + /// Dump every key/value pair across all column families into a [`DbDump`]. Column families are + /// discovered from disk, so new ones are captured without a hardcoded list. + pub fn dump_all(&self) -> DbResult { + let cf_names = + DBWithThreadMode::::list_cf(&Options::default(), self.db.path()) + .map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some("Failed to list column families for dump".to_owned()), + ) + })?; + + let mut entries = Vec::new(); + for cf_name in cf_names { + let cf = self.db.cf_handle(&cf_name).ok_or_else(|| { + DbError::db_interaction_error(format!( + "Column family {cf_name:?} listed on disk but not opened; add it to `open_inner`" + )) + })?; + for item in self.db.iterator_cf(&cf, IteratorMode::Start) { + let (key, value) = item.map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some(format!( + "Failed to iterate column family {cf_name:?} for dump" + )), + ) + })?; + entries.push(DbDumpEntry { + cf_name: cf_name.clone(), + key: key.into_vec(), + value: value.into_vec(), + }); + } + } + Ok(DbDump { entries }) + } + + /// Create a fresh rocksdb at `path` populated from a [`DbDump`]. + pub fn restore_from_dump(path: &Path, dump: &DbDump) -> DbResult { + let mut db_opts = Options::default(); + db_opts.create_missing_column_families(true); + db_opts.create_if_missing(true); + let dbio = Self::open_inner(path, &db_opts)?; + + let mut batch = WriteBatch::default(); + for entry in &dump.entries { + let cf = dbio.db.cf_handle(&entry.cf_name).ok_or_else(|| { + DbError::db_interaction_error(format!( + "Unknown column family {:?} in dump", + entry.cf_name + )) + })?; + batch.put_cf(&cf, &entry.key, &entry.value); + } + dbio.db.write(batch).map_err(|rerr| { + DbError::rocksdb_cast_message( + rerr, + Some("Failed to write dump restore batch".to_owned()), + ) + })?; + + Ok(dbio) + } + fn open_inner(path: &Path, db_opts: &Options) -> DbResult { let mut cf_opts = Options::default(); cf_opts.set_max_write_buffer_number(16); @@ -246,6 +354,11 @@ impl RocksDBIO { self.put(&ZoneSdkCheckpointCellRef(bytes), ()) } + /// Remove the persisted zone-sdk checkpoint so the next startup is treated as a fresh start. + pub fn delete_zone_sdk_checkpoint_bytes(&self) -> DbResult<()> { + self.del::(()) + } + pub fn get_pending_deposit_events(&self) -> DbResult> { Ok(self .get_opt::(())? @@ -448,10 +561,29 @@ impl RocksDBIO { } pub fn mark_block_as_finalized(&self, block_id: u64) -> DbResult<()> { + self.set_block_bedrock_status(block_id, BedrockStatus::Finalized) + } + + /// Reset every stored block to [`BedrockStatus::Pending`], for snapshotting a store to replay + /// against a fresh Bedrock instance that knows none of the blocks yet. + pub fn reset_all_blocks_to_pending(&self) -> DbResult<()> { + let block_ids: Vec = self + .get_all_blocks() + .filter_map(Result::ok) + .filter(|block| !matches!(block.bedrock_status, BedrockStatus::Pending)) + .map(|block| block.header.block_id) + .collect(); + for id in block_ids { + self.set_block_bedrock_status(id, BedrockStatus::Pending)?; + } + Ok(()) + } + + fn set_block_bedrock_status(&self, block_id: u64, status: BedrockStatus) -> DbResult<()> { let mut block = self.get_block(block_id)?.ok_or_else(|| { DbError::db_interaction_error(format!("Block with id {block_id} not found")) })?; - block.bedrock_status = BedrockStatus::Finalized; + block.bedrock_status = status; let cf_block = self.block_column(); self.db @@ -473,7 +605,7 @@ impl RocksDBIO { .map_err(|rerr| { DbError::rocksdb_cast_message( rerr, - Some(format!("Failed to mark block {block_id} as finalized")), + Some(format!("Failed to set block {block_id} bedrock status")), ) })?; diff --git a/lez/testnet_initial_state/src/lib.rs b/lez/testnet_initial_state/src/lib.rs index b608bb59..c92371cd 100644 --- a/lez/testnet_initial_state/src/lib.rs +++ b/lez/testnet_initial_state/src/lib.rs @@ -86,6 +86,7 @@ pub struct PublicAccountPublicInitialData { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PrivateAccountPublicInitialData { pub npk: lee_core::NullifierPublicKey, + pub vpk: lee_core::encryption::ViewingPublicKey, pub account: lee_core::account::Account, } @@ -108,6 +109,7 @@ impl PrivateAccountPrivateInitialData { pub fn account_id(&self) -> lee::AccountId { lee::AccountId::for_regular_private_account( &self.key_chain.nullifier_public_key, + &self.key_chain.viewing_public_key, self.identifier, ) } @@ -183,6 +185,7 @@ fn initial_commitments() -> Vec { .into_iter() .map(|data| PrivateAccountPublicInitialData { npk: data.key_chain.nullifier_public_key, + vpk: data.key_chain.viewing_public_key.clone(), account: data.account, }) .collect() @@ -193,7 +196,8 @@ fn initial_private_accounts() -> Vec<(lee_core::Commitment, lee_core::Nullifier) .iter() .map(|init_comm_data| { let npk = &init_comm_data.npk; - let account_id = lee::AccountId::for_regular_private_account(npk, 0); + let account_id = + lee::AccountId::for_regular_private_account(npk, &init_comm_data.vpk, 0); let mut acc = init_comm_data.account.clone(); @@ -331,8 +335,8 @@ mod tests { const PUB_ACC_A_TEXT_ADDR: &str = "6iArKUXxhUJqS7kCaPNhwMWt3ro71PDyBj7jwAyE2VQV"; const PUB_ACC_B_TEXT_ADDR: &str = "7wHg9sbJwc6h3NP1S9bekfAzB8CHifEcxKswCKUt3YQo"; - const PRIV_ACC_A_TEXT_ADDR: &str = "4eGX3M3rgjHsme8n3sSp89af8JRZtYVTesbJjLqaX1VQ"; - const PRIV_ACC_B_TEXT_ADDR: &str = "3m6HQmCgmAvsxZtxAHPqqEqoBG4335fCG8TzxigyW7rE"; + const PRIV_ACC_A_TEXT_ADDR: &str = "EVesBKsYRVtkjnTcsbk8tWHkBn2xZmzAXzwgrP3ZaVoZ"; + const PRIV_ACC_B_TEXT_ADDR: &str = "94MXhZnueurjX6v37CYDKVEKYBiyhYArvtEdceq2XDQP"; #[test] fn pub_state_consistency() { @@ -467,6 +471,10 @@ mod tests { init_comms[0], PrivateAccountPublicInitialData { npk: NullifierPublicKey(NPK_PRIV_ACC_A), + vpk: init_private_accs_keys[0] + .key_chain + .viewing_public_key + .clone(), account: Account { program_owner: DEFAULT_PROGRAM_OWNER, balance: PRIV_ACC_A_INITIAL_BALANCE, @@ -480,6 +488,10 @@ mod tests { init_comms[1], PrivateAccountPublicInitialData { npk: NullifierPublicKey(NPK_PRIV_ACC_B), + vpk: init_private_accs_keys[1] + .key_chain + .viewing_public_key + .clone(), account: Account { program_owner: DEFAULT_PROGRAM_OWNER, balance: PRIV_ACC_B_INITIAL_BALANCE, diff --git a/lez/wallet-ffi/Cargo.toml b/lez/wallet-ffi/Cargo.toml index 47d18c53..b5fe7635 100644 --- a/lez/wallet-ffi/Cargo.toml +++ b/lez/wallet-ffi/Cargo.toml @@ -28,9 +28,6 @@ vault_core.workspace = true [build-dependencies] cbindgen = "0.29" -[dev-dependencies] -tempfile = "3" - [features] default = ["prove"] prove = ["lee/prove"] diff --git a/lez/wallet-ffi/src/types.rs b/lez/wallet-ffi/src/types.rs index 98590619..4f041c55 100644 --- a/lez/wallet-ffi/src/types.rs +++ b/lez/wallet-ffi/src/types.rs @@ -645,9 +645,10 @@ mod tests { let identifier = u128::from_le_bytes([45; 16]); let private_reg_acc_id = - AccountId::for_private_account(&npk, &PrivateAccountKind::Regular(identifier)); + AccountId::for_private_account(&npk, &vpk, &PrivateAccountKind::Regular(identifier)); let private_pda_acc_id = AccountId::for_private_account( &npk, + &vpk, &PrivateAccountKind::Pda { program_id: [46; 8], seed: PdaSeed::new([47; 32]), diff --git a/lez/wallet/src/account_manager.rs b/lez/wallet/src/account_manager.rs index dae8fd31..a5f900d6 100644 --- a/lez/wallet/src/account_manager.rs +++ b/lez/wallet/src/account_manager.rs @@ -1,15 +1,15 @@ use core::fmt; use anyhow::Result; -use key_protocol::key_management::ephemeral_key_holder::EphemeralKeyHolder; use keycard_wallet::{KeycardWallet, python_path}; use lee::{AccountId, PrivateKey, PublicKey, Signature}; use lee_core::{ CommitmentSetDigest, DUMMY_COMMITMENT_HASH, Identifier, InputAccountIdentity, MembershipProof, NullifierPublicKey, NullifierSecretKey, SharedSecretKey, account::{AccountWithMetadata, Nonce}, - encryption::{EncryptedAccountData, EphemeralPublicKey, ViewingPublicKey}, + encryption::ViewingPublicKey, }; +use rand::{RngCore as _, rngs::OsRng}; use crate::{ExecutionFailureKind, WalletCore}; @@ -201,15 +201,38 @@ impl AccountManager { for account in accounts { let state = match account { AccountIdentity::Public(account_id) => { - prepare_public_state(wallet, account_id, true).await? + let acc = wallet + .get_account_public(account_id) + .await + .map_err(ExecutionFailureKind::SequencerError)?; + + let sk = wallet.get_account_public_signing_key(account_id).cloned(); + let account = AccountWithMetadata::new(acc.clone(), sk.is_some(), account_id); + + State::Public { account, sk } } AccountIdentity::PublicNoSign(account_id) => { - prepare_public_state(wallet, account_id, false).await? + let acc = wallet + .get_account_public(account_id) + .await + .map_err(ExecutionFailureKind::SequencerError)?; + + let sk = None; + let account = AccountWithMetadata::new(acc.clone(), sk.is_some(), account_id); + + State::Public { account, sk } } AccountIdentity::PublicKeycard { account_id, key_path, } => { + let acc = wallet + .get_account_public(account_id) + .await + .map_err(ExecutionFailureKind::SequencerError)?; + + let account = AccountWithMetadata::new(acc.clone(), true, account_id); + if pin.is_none() { pin = Some( crate::helperfunctions::read_pin() @@ -225,40 +248,75 @@ impl AccountManager { .to_owned(), ); } - prepare_public_keycard_state(wallet, account_id, key_path).await? + + State::PublicKeycard { account, key_path } + } + AccountIdentity::PrivateOwned(account_id) => { + let pre = private_key_tree_acc_preparation(wallet, account_id, false).await?; + + State::Private(pre) } - AccountIdentity::PrivateOwned(account_id) => State::Private( - private_key_tree_acc_preparation(wallet, account_id, false).await?, - ), AccountIdentity::PrivateForeign { npk, vpk, identifier, - } => State::Private(private_foreign_acc_preparation(npk, vpk, identifier)), - AccountIdentity::PrivatePdaOwned(account_id) => State::Private( - private_key_tree_acc_preparation(wallet, account_id, true).await?, - ), + } => { + let acc = lee_core::account::Account::default(); + let auth_acc = AccountWithMetadata::new(acc, false, (&npk, &vpk, identifier)); + let mut random_seed: [u8; 32] = [0; 32]; + OsRng.fill_bytes(&mut random_seed); + let pre = AccountPreparedData { + nsk: None, + npk, + identifier, + vpk, + pre_state: auth_acc, + proof: None, + random_seed, + is_pda: false, + }; + + State::Private(pre) + } + AccountIdentity::PrivatePdaOwned(account_id) => { + let pre = private_key_tree_acc_preparation(wallet, account_id, true).await?; + State::Private(pre) + } AccountIdentity::PrivatePdaForeign { account_id, npk, vpk, identifier, - } => State::Private(private_pda_foreign_acc_preparation( - account_id, npk, vpk, identifier, - )), + } => { + let acc = lee_core::account::Account::default(); + let auth_acc = AccountWithMetadata::new(acc, false, account_id); + let mut random_seed: [u8; 32] = [0; 32]; + OsRng.fill_bytes(&mut random_seed); + let pre = AccountPreparedData { + nsk: None, + npk, + identifier, + vpk, + pre_state: auth_acc, + proof: None, + random_seed, + is_pda: true, + }; + State::Private(pre) + } AccountIdentity::PrivateShared { nsk, npk, vpk, identifier, } => { - let account_id = lee::AccountId::from((&npk, identifier)); - State::Private( - private_shared_acc_preparation( - wallet, account_id, nsk, npk, vpk, identifier, false, - ) - .await?, + let account_id = lee::AccountId::from((&npk, &vpk, identifier)); + let pre = private_shared_acc_preparation( + wallet, account_id, nsk, npk, vpk, identifier, false, ) + .await?; + + State::Private(pre) } AccountIdentity::PrivatePdaShared { account_id, @@ -266,12 +324,14 @@ impl AccountManager { npk, vpk, identifier, - } => State::Private( - private_shared_acc_preparation( + } => { + let pre = private_shared_acc_preparation( wallet, account_id, nsk, npk, vpk, identifier, true, ) - .await?, - ), + .await?; + + State::Private(pre) + } }; states.push(state); @@ -327,9 +387,26 @@ impl AccountManager { self.states .iter() .filter_map(|state| match state { - State::Private(pre) => Some(PrivateAccountKeys { ssk: pre.ssk }), + State::Private(pre) => Some(pre), State::Public { .. } | State::PublicKeycard { .. } => None, }) + .map(|pre| { + let nonce = if pre.proof.is_some() { + pre.pre_state.account.nonce.private_account_nonce_increment( + pre.nsk.as_ref().expect("update variant must have nsk"), + ) + } else { + lee_core::account::Nonce::private_account_nonce_init(&pre.pre_state.account_id) + }; + let esk = lee_core::EphemeralSecretKey::new( + &pre.pre_state.account_id, + &pre.random_seed, + &nonce, + ); + PrivateAccountKeys { + ssk: SharedSecretKey::encapsulate_deterministic(&pre.vpk, &esk).0, + } + }) .collect() } @@ -344,19 +421,17 @@ impl AccountManager { State::Public { .. } | State::PublicKeycard { .. } => InputAccountIdentity::Public, State::Private(pre) if pre.is_pda => match (pre.nsk, pre.proof.clone()) { (Some(nsk), Some(membership_proof)) => InputAccountIdentity::PrivatePdaUpdate { - epk: pre.epk.clone(), - view_tag: EncryptedAccountData::compute_view_tag(&pre.npk, &pre.vpk), - ssk: pre.ssk, + vpk: pre.vpk.clone(), + random_seed: pre.random_seed, nsk, membership_proof, identifier: pre.identifier, seed: None, }, _ => InputAccountIdentity::PrivatePdaInit { - epk: pre.epk.clone(), - view_tag: EncryptedAccountData::compute_view_tag(&pre.npk, &pre.vpk), + vpk: pre.vpk.clone(), + random_seed: pre.random_seed, npk: pre.npk, - ssk: pre.ssk, identifier: pre.identifier, commitment_root: self.dummy_commitment_root, seed: None, @@ -365,27 +440,24 @@ impl AccountManager { State::Private(pre) => match (pre.nsk, pre.proof.clone()) { (Some(nsk), Some(membership_proof)) => { InputAccountIdentity::PrivateAuthorizedUpdate { - epk: pre.epk.clone(), - view_tag: EncryptedAccountData::compute_view_tag(&pre.npk, &pre.vpk), - ssk: pre.ssk, + vpk: pre.vpk.clone(), + random_seed: pre.random_seed, nsk, membership_proof, identifier: pre.identifier, } } (Some(nsk), None) => InputAccountIdentity::PrivateAuthorizedInit { - epk: pre.epk.clone(), - view_tag: EncryptedAccountData::compute_view_tag(&pre.npk, &pre.vpk), - ssk: pre.ssk, + vpk: pre.vpk.clone(), + random_seed: pre.random_seed, nsk, identifier: pre.identifier, commitment_root: self.dummy_commitment_root, }, (None, _) => InputAccountIdentity::PrivateUnauthorized { - epk: pre.epk.clone(), - view_tag: EncryptedAccountData::compute_view_tag(&pre.npk, &pre.vpk), + vpk: pre.vpk.clone(), + random_seed: pre.random_seed, npk: pre.npk, - ssk: pre.ssk, identifier: pre.identifier, commitment_root: self.dummy_commitment_root, }, @@ -462,49 +534,12 @@ struct AccountPreparedData { vpk: ViewingPublicKey, pre_state: AccountWithMetadata, proof: Option, - /// Cached shared-secret key derived once at `AccountManager::new`. Reused for both the - /// circuit input variant (`account_identities()`) and the message ephemeral-key tuples - /// (`private_account_keys()`), so all consumers see the same key. The corresponding - /// `EphemeralKeyHolder` uses `OsRng` and would produce a different value on a second call. - ssk: SharedSecretKey, - /// Cached ephemeral public key, paired with `ssk`. - epk: EphemeralPublicKey, + random_seed: [u8; 32], /// True when this account is a private PDA (owned or foreign). Used by `account_identities()` /// to select `PrivatePdaInit`/`PrivatePdaUpdate` rather than the standalone private variants. is_pda: bool, } -async fn prepare_public_state( - wallet: &WalletCore, - account_id: AccountId, - lookup_signing_key: bool, -) -> Result { - let acc = wallet - .get_account_public(account_id) - .await - .map_err(ExecutionFailureKind::SequencerError)?; - let sk = if lookup_signing_key { - wallet.get_account_public_signing_key(account_id).cloned() - } else { - None - }; - let account = AccountWithMetadata::new(acc, sk.is_some(), account_id); - Ok(State::Public { account, sk }) -} - -async fn prepare_public_keycard_state( - wallet: &WalletCore, - account_id: AccountId, - key_path: String, -) -> Result { - let acc = wallet - .get_account_public(account_id) - .await - .map_err(ExecutionFailureKind::SequencerError)?; - let account = AccountWithMetadata::new(acc, true, account_id); - Ok(State::PublicKeycard { account, key_path }) -} - async fn private_key_tree_acc_preparation( wallet: &WalletCore, account_id: AccountId, @@ -530,9 +565,8 @@ async fn private_key_tree_acc_preparation( // support from that in the wallet. let sender_pre = AccountWithMetadata::new(from_acc.account.clone(), true, account_id); - let eph_holder = EphemeralKeyHolder::new(&from_vpk); - let ssk = eph_holder.calculate_shared_secret_sender(); - let epk = eph_holder.ephemeral_public_key().clone(); + let mut random_seed: [u8; 32] = [0; 32]; + OsRng.fill_bytes(&mut random_seed); Ok(AccountPreparedData { nsk: Some(nsk), @@ -541,8 +575,7 @@ async fn private_key_tree_acc_preparation( vpk: from_vpk, pre_state: sender_pre, proof, - ssk, - epk, + random_seed, is_pda, }) } @@ -570,9 +603,8 @@ async fn private_shared_acc_preparation( .await .unwrap_or(None); - let eph_holder = EphemeralKeyHolder::new(&vpk); - let ssk = eph_holder.calculate_shared_secret_sender(); - let epk = eph_holder.ephemeral_public_key().clone(); + let mut random_seed: [u8; 32] = [0; 32]; + OsRng.fill_bytes(&mut random_seed); Ok(AccountPreparedData { nsk: Some(nsk), @@ -581,61 +613,11 @@ async fn private_shared_acc_preparation( vpk, pre_state, proof, - ssk, - epk, + random_seed, is_pda, }) } -fn private_foreign_acc_preparation( - npk: NullifierPublicKey, - vpk: ViewingPublicKey, - identifier: Identifier, -) -> AccountPreparedData { - let acc = lee_core::account::Account::default(); - let pre_state = AccountWithMetadata::new(acc, false, (&npk, identifier)); - let eph_holder = EphemeralKeyHolder::new(&vpk); - let ssk = eph_holder.calculate_shared_secret_sender(); - let epk = eph_holder.ephemeral_public_key().clone(); - - AccountPreparedData { - nsk: None, - npk, - identifier, - vpk, - pre_state, - proof: None, - ssk, - epk, - is_pda: false, - } -} - -fn private_pda_foreign_acc_preparation( - account_id: AccountId, - npk: NullifierPublicKey, - vpk: ViewingPublicKey, - identifier: Identifier, -) -> AccountPreparedData { - let acc = lee_core::account::Account::default(); - let pre_state = AccountWithMetadata::new(acc, false, account_id); - let eph_holder = EphemeralKeyHolder::new(&vpk); - let ssk = eph_holder.calculate_shared_secret_sender(); - let epk = eph_holder.ephemeral_public_key().clone(); - - AccountPreparedData { - nsk: None, - npk, - identifier, - vpk, - pre_state, - proof: None, - ssk, - epk, - is_pda: true, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/lez/wallet/src/cli/account.rs b/lez/wallet/src/cli/account.rs index b49cfb25..275e13fc 100644 --- a/lez/wallet/src/cli/account.rs +++ b/lez/wallet/src/cli/account.rs @@ -581,8 +581,11 @@ impl WalletSubcommand for ImportSubcommand { let key_chain: KeyChain = serde_json::from_str(&key_chain_json) .map_err(|err| anyhow::anyhow!("Invalid key chain JSON: {err}"))?; let account = lee::Account::from(account_state); - let account_id = - lee::AccountId::from((&key_chain.nullifier_public_key, identifier)); + let account_id = lee::AccountId::from(( + &key_chain.nullifier_public_key, + &key_chain.viewing_public_key, + identifier, + )); wallet_core .storage_mut() diff --git a/lez/wallet/src/cli/mod.rs b/lez/wallet/src/cli/mod.rs index 43b6dc7a..44b578a1 100644 --- a/lez/wallet/src/cli/mod.rs +++ b/lez/wallet/src/cli/mod.rs @@ -7,6 +7,7 @@ use common::{HashType, transaction::LeeTransaction}; use derive_more::Display; use futures::TryFutureExt as _; use lee::ProgramDeploymentTransaction; +use lee_core::BlockId; use sequencer_service_rpc::RpcClient as _; pub use crate::helperfunctions::{read_mnemonic, read_pin}; @@ -117,11 +118,11 @@ pub struct Args { #[derive(Debug, Clone)] pub enum SubcommandReturnValue { - PrivacyPreservingTransfer { tx_hash: HashType }, + TransactionExecuted { tx_hash: HashType }, RegisterAccount { account_id: lee::AccountId }, Account(lee::Account), Empty, - SyncedToBlock(u64), + SyncedToBlock(BlockId), } #[derive(Debug, Display, Clone, PartialEq, Eq, Hash)] @@ -286,13 +287,16 @@ pub async fn execute_subcommand( ))?; let message = lee::program_deployment_transaction::Message::new(bytecode); let transaction = ProgramDeploymentTransaction::new(message); - let _response = wallet_core + let tx_hash = wallet_core .sequencer_client .send_transaction(LeeTransaction::ProgramDeployment(transaction)) .await .context("Transaction submission error")?; - SubcommandReturnValue::Empty + wallet_core + .poll_and_finalize_public_transaction(tx_hash) + .await + .context("Transaction finalization error")? } }; diff --git a/lez/wallet/src/cli/programs/bridge.rs b/lez/wallet/src/cli/programs/bridge.rs index 8d8cf5ee..302ffbfb 100644 --- a/lez/wallet/src/cli/programs/bridge.rs +++ b/lez/wallet/src/cli/programs/bridge.rs @@ -47,9 +47,10 @@ impl WalletSubcommand for BridgeSubcommand { .send_withdraw(sender_account_id, amount, bedrock_account_pk) .await?; - println!("Transaction hash is {tx_hash}"); - - Ok(SubcommandReturnValue::Empty) + wallet_core + .poll_and_finalize_public_transaction(tx_hash) + .await + .context("Transaction finalization error") } } } diff --git a/lez/wallet/src/cli/programs/native_token_transfer.rs b/lez/wallet/src/cli/programs/native_token_transfer.rs index ccc2f14c..aaa3efe1 100644 --- a/lez/wallet/src/cli/programs/native_token_transfer.rs +++ b/lez/wallet/src/cli/programs/native_token_transfer.rs @@ -443,7 +443,7 @@ impl NativeTokenTransferProgramSubcommandShielded { wallet_core.store_persistent_data()?; - Ok(SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash }) + Ok(SubcommandReturnValue::TransactionExecuted { tx_hash }) } } diff --git a/lez/wallet/src/lib.rs b/lez/wallet/src/lib.rs index 923f55ae..5a69d994 100644 --- a/lez/wallet/src/lib.rs +++ b/lez/wallet/src/lib.rs @@ -377,7 +377,7 @@ impl WalletCore { let keys = holder.derive_keys_for_pda(&program_id, &pda_seed); let npk = keys.generate_nullifier_public_key(); let vpk = keys.generate_viewing_public_key(); - let account_id = AccountId::for_private_pda(&program_id, &pda_seed, &npk, identifier); + let account_id = AccountId::for_private_pda(&program_id, &pda_seed, &npk, &vpk, identifier); self.register_shared_account( account_id, @@ -419,7 +419,7 @@ impl WalletCore { let keys = holder.derive_keys_for_shared_account(&derivation_seed); let npk = keys.generate_nullifier_public_key(); let vpk = keys.generate_viewing_public_key(); - let account_id = AccountId::from((&npk, identifier)); + let account_id = AccountId::from((&npk, &vpk, identifier)); self.register_shared_account(account_id, group_name, identifier, None, None); @@ -490,11 +490,6 @@ impl WalletCore { Some(Commitment::new(&account_id, account)) } - /// Poll transactions. - pub async fn poll_native_token_transfer(&self, hash: HashType) -> Result { - self.poller.poll_tx(hash).await - } - pub async fn check_private_account_initialized( &self, account_id: AccountId, @@ -549,7 +544,6 @@ impl WalletCore { } } - println!("Transaction data is {:?}", tx.message); Ok(()) } @@ -558,10 +552,11 @@ impl WalletCore { tx_hash: HashType, ) -> Result { println!("Transaction hash is {tx_hash}"); - let transfer_tx = self.poll_native_token_transfer(tx_hash).await?; - println!("Transaction data is {transfer_tx:?}"); + let (tx, block_id) = self.poller.poll_tx(tx_hash).await?; + println!("Transaction is included in block {block_id}"); + println!("Transaction data is {tx:?}"); self.store_persistent_data()?; - Ok(cli::SubcommandReturnValue::Empty) + Ok(cli::SubcommandReturnValue::TransactionExecuted { tx_hash }) } /// Pass an empty slice when the recipient is foreign and no accounts need decoding. @@ -571,12 +566,17 @@ impl WalletCore { acc_decode_data: &[AccDecodeData], ) -> Result { println!("Transaction hash is {tx_hash}"); - let transfer_tx = self.poll_native_token_transfer(tx_hash).await?; - if let common::transaction::LeeTransaction::PrivacyPreserving(tx) = transfer_tx { - self.decode_insert_privacy_preserving_transaction_results(&tx, acc_decode_data)?; + let (tx, block_id) = self.poller.poll_tx(tx_hash).await?; + println!("Transaction is included in block {block_id}"); + println!("Transaction data is {tx:?}"); + if let common::transaction::LeeTransaction::PrivacyPreserving(private_tx) = tx { + self.decode_insert_privacy_preserving_transaction_results( + &private_tx, + acc_decode_data, + )?; } self.store_persistent_data()?; - Ok(cli::SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash }) + Ok(cli::SubcommandReturnValue::TransactionExecuted { tx_hash }) } pub async fn send_privacy_preserving_tx( @@ -796,7 +796,11 @@ impl WalletCore { ) .map(|(kind, res_acc)| { let npk = &key_chain.nullifier_public_key; - let account_id = lee::AccountId::for_private_account(npk, &kind); + let account_id = lee::AccountId::for_private_account( + npk, + &key_chain.viewing_public_key, + &kind, + ); (account_id, kind, res_acc) }) }) diff --git a/lez/wallet/src/poller.rs b/lez/wallet/src/poller.rs index 52802887..63ba2275 100644 --- a/lez/wallet/src/poller.rs +++ b/lez/wallet/src/poller.rs @@ -2,6 +2,7 @@ use std::time::Duration; use anyhow::Result; use common::{HashType, block::Block, transaction::LeeTransaction}; +use lee_core::BlockId; use log::{info, warn}; use sequencer_service_rpc::{RpcClient as _, SequencerClient}; @@ -30,7 +31,7 @@ impl TxPoller { } // TODO: this polling is not based on blocks, but on timeouts, need to fix this. - pub async fn poll_tx(&self, tx_hash: HashType) -> Result { + pub async fn poll_tx(&self, tx_hash: HashType) -> Result<(LeeTransaction, BlockId)> { let max_blocks_to_query = self.polling_max_blocks_to_query; info!("Starting poll for transaction {tx_hash}"); @@ -41,7 +42,7 @@ impl TxPoller { loop { match self.client.get_transaction(tx_hash).await { - Ok(Some(tx)) => return Ok(tx), + Ok(Some((tx, block_id))) => return Ok((tx, block_id)), Ok(None) => {} Err(err) => { warn!("Failed to get transaction by hash {tx_hash} with error: {err:#?}"); diff --git a/lez/wallet/src/storage/key_chain.rs b/lez/wallet/src/storage/key_chain.rs index f73833eb..5db90119 100644 --- a/lez/wallet/src/storage/key_chain.rs +++ b/lez/wallet/src/storage/key_chain.rs @@ -174,7 +174,8 @@ impl UserKeyChain { .1 .first_key_value() .expect("Newly created key chain node must have at least one account"); - let account_id = AccountId::for_private_account(&npk, kind); + let account_id = + AccountId::for_private_account(&npk, &node.value.0.viewing_public_key, kind); (account_id, chain_index) } @@ -240,6 +241,7 @@ impl UserKeyChain { .find_map(|found| { let expected_id = AccountId::for_private_account( &found.key_chain.nullifier_public_key, + &found.key_chain.viewing_public_key, found.kind, ); (expected_id == account_id).then_some(found) @@ -264,8 +266,11 @@ impl UserKeyChain { .iter() .flat_map(|(key, data)| { data.accounts.keys().map(|kind| { - let account_id = - AccountId::for_private_account(&key.key_chain.nullifier_public_key, kind); + let account_id = AccountId::for_private_account( + &key.key_chain.nullifier_public_key, + &key.key_chain.viewing_public_key, + kind, + ); (account_id, &key.key_chain, key.chain_index.as_ref()) }) }) @@ -340,8 +345,11 @@ impl UserKeyChain { // Then try to update imported account for (key, data) in &mut self.imported_private_accounts { for (kind, imported_account) in &mut data.accounts { - let expected_id = - AccountId::for_private_account(&key.key_chain.nullifier_public_key, kind); + let expected_id = AccountId::for_private_account( + &key.key_chain.nullifier_public_key, + &key.key_chain.viewing_public_key, + kind, + ); if expected_id == account_id { debug!("Updating imported private account {account_id}"); *imported_account = account; @@ -378,8 +386,11 @@ impl UserKeyChain { // Node not yet in account_id_map โ€” find it by checking all nodes for (ci, node) in &mut self.private_key_tree.key_map { - let expected_id = - lee::AccountId::for_private_account(&node.value.0.nullifier_public_key, &kind); + let expected_id = lee::AccountId::for_private_account( + &node.value.0.nullifier_public_key, + &node.value.0.viewing_public_key, + &kind, + ); if expected_id == account_id { match node.value.1.entry(kind) { Entry::Occupied(mut occupied) => { @@ -429,8 +440,11 @@ impl UserKeyChain { .iter() .flat_map(|(key, data)| { data.accounts.keys().map(|kind| { - let account_id = - AccountId::for_private_account(&key.key_chain.nullifier_public_key, kind); + let account_id = AccountId::for_private_account( + &key.key_chain.nullifier_public_key, + &key.key_chain.viewing_public_key, + kind, + ); (account_id, key.chain_index.as_ref()) }) }) @@ -720,7 +734,11 @@ mod tests { let mut user_data = UserKeyChain::default(); let key_chain = KeyChain::new_os_random(); - let account_id = AccountId::from((&key_chain.nullifier_public_key, 0)); + let account_id = AccountId::from(( + &key_chain.nullifier_public_key, + &key_chain.viewing_public_key, + 0, + )); let account = lee_core::account::Account::default(); user_data.add_imported_private_account(key_chain, None, 0, account); @@ -735,7 +753,11 @@ mod tests { let mut user_data = UserKeyChain::default(); let key_chain = KeyChain::new_os_random(); - let account_id = AccountId::from((&key_chain.nullifier_public_key, 0)); + let account_id = AccountId::from(( + &key_chain.nullifier_public_key, + &key_chain.viewing_public_key, + 0, + )); let account = lee_core::account::Account::default(); user_data.add_imported_private_account(key_chain, None, 0, account.clone()); @@ -780,7 +802,11 @@ mod tests { let mut user_data = UserKeyChain::default(); let key_chain = KeyChain::new_os_random(); - let account_id = AccountId::from((&key_chain.nullifier_public_key, 0)); + let account_id = AccountId::from(( + &key_chain.nullifier_public_key, + &key_chain.viewing_public_key, + 0, + )); let new_account = lee_core::account::Account { balance: 100, @@ -801,7 +827,11 @@ mod tests { let mut user_data = UserKeyChain::default(); let key_chain = KeyChain::new_os_random(); - let account_id1 = AccountId::from((&key_chain.nullifier_public_key, 0)); + let account_id1 = AccountId::from(( + &key_chain.nullifier_public_key, + &key_chain.viewing_public_key, + 0, + )); let account = lee_core::account::Account::default(); user_data.add_imported_private_account(key_chain, None, 0, account); diff --git a/test_fixtures/Cargo.toml b/test_fixtures/Cargo.toml index 71f887b6..88526dea 100644 --- a/test_fixtures/Cargo.toml +++ b/test_fixtures/Cargo.toml @@ -21,6 +21,7 @@ wallet.workspace = true programs.workspace = true anyhow.workspace = true +bip39.workspace = true bytesize.workspace = true env_logger.workspace = true futures.workspace = true @@ -31,4 +32,5 @@ serde_json.workspace = true tempfile.workspace = true testcontainers = { version = "0.27.3", features = ["docker-compose"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +tokio-util.workspace = true url.workspace = true diff --git a/test_fixtures/README.md b/test_fixtures/README.md new file mode 100644 index 00000000..502ed8b5 --- /dev/null +++ b/test_fixtures/README.md @@ -0,0 +1,31 @@ +# test_fixtures + +Shared test/bench fixtures that spin up bedrock + sequencer + indexer + wallet end-to-end. + +## Library + +`TestContext` drives the full stack for integration tests and benches: + +```rust +let ctx = TestContext::new().await?; // fast: restores the prebuilt db dump +let ctx = TestContext::builder() + .from_scratch() // apply genesis + claim supply live + .with_genesis(actions) // custom genesis (implies from_scratch) + .disable_indexer() + .build().await?; +``` + +`TestContext::new()` restores `fixtures/prebuilt_sequencer_db.dump` (genesis + initial-supply claim +blocks) instead of re-applying genesis and re-proving the private claims, then syncs the wallet. Use +`from_scratch()` to build genesis live. + +## Binary + +`regenerate_test_fixture` regenerates that dump (needs Docker): + +```sh +just regenerate-test-fixture # RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture +``` + +Rerun and commit `fixtures/prebuilt_sequencer_db.dump` after changing genesis, the default accounts, +block format, or initial state. diff --git a/test_fixtures/fixtures/prebuilt_sequencer_db.dump b/test_fixtures/fixtures/prebuilt_sequencer_db.dump new file mode 100644 index 00000000..fcaeac6f Binary files /dev/null and b/test_fixtures/fixtures/prebuilt_sequencer_db.dump differ diff --git a/test_fixtures/src/bin/regenerate_test_fixture.rs b/test_fixtures/src/bin/regenerate_test_fixture.rs new file mode 100644 index 00000000..80da8fdc --- /dev/null +++ b/test_fixtures/src/bin/regenerate_test_fixture.rs @@ -0,0 +1,126 @@ +//! Regenerate the prebuilt sequencer database dump for the fast `TestContext::new()` path. +//! Needs Docker. Run via `just regenerate-test-fixture`, then commit the dump. + +#![expect(clippy::print_stdout, reason = "It's normal in this small cli")] + +use std::{path::Path, time::Duration}; + +use anyhow::{Context as _, Result}; +use lee::PrivateKey; +use sequencer_core::block_store::SequencerStore; +use test_fixtures::{ + config, + setup::{ + prebuilt_sequencer_db_dump_path, setup_bedrock_node, + setup_private_accounts_with_initial_supply, setup_public_accounts_with_initial_supply, + setup_sequencer, setup_wallet, + }, +}; +use wallet::config::WalletConfigOverrides; + +#[tokio::main] +async fn main() -> Result<()> { + env_logger::init(); + + let dest = prebuilt_sequencer_db_dump_path(); + println!( + "๐Ÿ—ƒ๏ธ Regenerating prebuilt sequencer db fixture at {}", + dest.display() + ); + + generate_prebuilt_fixture(&dest) + .await + .context("Failed to regenerate prebuilt sequencer database fixture")?; + + println!("โœ… Wrote fixture dump to {}", dest.display()); + Ok(()) +} + +/// Run a real sequencer with the default accounts, apply genesis + claim the initial supply +/// (genesis block + claim block), then strip the checkpoint and reset blocks to `Pending` so the +/// dump replays cleanly against a fresh Bedrock. Writes the dump to `dest`. +async fn generate_prebuilt_fixture(dest: &Path) -> Result<()> { + let (_bedrock_compose, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to setup Bedrock node")?; + + let initial_public_accounts = config::default_public_accounts_for_wallet(); + let initial_private_accounts = config::default_private_accounts_for_wallet(); + let genesis = + config::genesis_from_accounts(&initial_public_accounts, &initial_private_accounts); + + let (sequencer_handle, temp_sequencer_dir) = setup_sequencer( + config::SequencerPartialConfig::default(), + bedrock_addr, + genesis, + config::bedrock_channel_id(), + None, + ) + .await + .context("Failed to setup Sequencer for fixture generation")?; + + let (mut wallet, _temp_wallet_dir, _wallet_password) = setup_wallet( + sequencer_handle.addr(), + &initial_public_accounts, + &initial_private_accounts, + WalletConfigOverrides::default(), + ) + .context("Failed to setup wallet for fixture generation")?; + + setup_public_accounts_with_initial_supply(&mut wallet, &initial_public_accounts) + .await + .context("Failed to initialize public accounts for fixture generation")?; + setup_private_accounts_with_initial_supply(&mut wallet, &initial_private_accounts) + .await + .context("Failed to initialize private accounts for fixture generation")?; + + // Shut down gracefully to release the rocksdb lock before reopening the store. + drop(wallet); + drop(sequencer_handle); + + let db_path = temp_sequencer_dir.path().join("rocksdb"); + let store = open_store_with_retry(&db_path) + .await + .context("Failed to reopen sequencer store after shutdown")?; + store + .delete_zone_checkpoint() + .context("Failed to strip zone-sdk checkpoint from fixture database")?; + store + .reset_all_blocks_to_pending() + .context("Failed to reset fixture blocks to pending")?; + let dump = store.dump().context("Failed to dump fixture database")?; + drop(store); + + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("Failed to create fixture directory {}", parent.display()))?; + } + let bytes = dump + .to_bytes() + .context("Failed to serialize fixture dump")?; + std::fs::write(dest, bytes) + .with_context(|| format!("Failed to write fixture dump to {}", dest.display()))?; + + Ok(()) +} + +/// Reopen the store, retrying while the shut-down sequencer's aborted tasks are still releasing +/// their db handles (the process holds the file lock until then). Must `await` between attempts. +async fn open_store_with_retry(db_path: &Path) -> Result { + let signing_key = PrivateKey::try_new(config::SEQUENCER_SIGNING_KEY) + .expect("Fixed sequencer signing key must be valid"); + + let mut last_err = None; + for _ in 0..100 { + // Let the runtime drop the aborted sequencer tasks before each attempt. + tokio::time::sleep(Duration::from_millis(100)).await; + match SequencerStore::open_db(db_path, signing_key.clone()) { + Ok(store) => return Ok(store), + Err(err) => last_err = Some(err), + } + } + Err(anyhow::anyhow!( + "Failed to open sequencer store at {} after retries: {last_err:?}", + db_path.display() + )) +} diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index ea52bf5b..960f349b 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -3,7 +3,7 @@ use std::{net::SocketAddr, path::PathBuf, time::Duration}; use anyhow::{Context as _, Result}; use bytesize::ByteSize; use indexer_service::{ChannelId, ClientConfig, IndexerConfig}; -use key_protocol::key_management::KeyChain; +use key_protocol::key_management::{KeyChain, secret_holders::SeedHolder}; use lee::{AccountId, PrivateKey, PublicKey}; use lee_core::Identifier; use sequencer_core::config::{BedrockConfig, CrossZoneConfig, GenesisAction, SequencerConfig}; @@ -13,6 +13,15 @@ use wallet::config::WalletConfig; pub const INITIAL_PUBLIC_BALANCES_FOR_WALLET: [u128; 2] = [10_000, 20_000]; pub const INITIAL_PRIVATE_BALANCES_FOR_WALLET: [u128; 2] = [10_000, 20_000]; +/// Fixed sequencer signing key; exposed so the fixture generator can reopen the produced store. +pub const SEQUENCER_SIGNING_KEY: [u8; 32] = [37; 32]; + +// Fixed entropy seeds for the default accounts: deterministic so one prebuilt database is reusable, +// and distinct from the `testnet_initial_state` accounts to avoid depending on / double-funding +// them. +const DEFAULT_PUBLIC_ACCOUNT_SEEDS: [[u8; 32]; 2] = [[0x11; 32], [0x22; 32]]; +const DEFAULT_PRIVATE_ACCOUNT_SEEDS: [[u8; 32]; 2] = [[0x33; 32], [0x44; 32]]; + #[derive(Clone)] pub struct InitialPrivateAccountForWallet { pub key_chain: KeyChain, @@ -23,7 +32,11 @@ pub struct InitialPrivateAccountForWallet { impl InitialPrivateAccountForWallet { #[must_use] pub fn account_id(&self) -> AccountId { - AccountId::from((&self.key_chain.nullifier_public_key, self.identifier)) + AccountId::from(( + &self.key_chain.nullifier_public_key, + &self.key_chain.viewing_public_key, + self.identifier, + )) } } @@ -85,7 +98,7 @@ pub fn sequencer_config( block_create_timeout, retry_pending_blocks_timeout: Duration::from_secs(5), genesis: genesis_transactions, - signing_key: [37; 32], + signing_key: SEQUENCER_SIGNING_KEY, bedrock_config: BedrockConfig { channel_id, node_url: addr_to_url(UrlProtocol::Http, bedrock_addr) @@ -98,7 +111,10 @@ pub fn sequencer_config( #[must_use] pub fn default_public_accounts_for_wallet() -> Vec<(PrivateKey, u128)> { - let mut private_keys = vec![PrivateKey::new_os_random(), PrivateKey::new_os_random()]; + let mut private_keys = DEFAULT_PUBLIC_ACCOUNT_SEEDS + .iter() + .map(|seed| PrivateKey::try_new(*seed).expect("Fixed public account seed must be valid")) + .collect::>(); private_keys.sort_unstable_by_key(|private_key| { AccountId::from(&PublicKey::new_from_private_key(private_key)) }); @@ -111,7 +127,10 @@ pub fn default_public_accounts_for_wallet() -> Vec<(PrivateKey, u128)> { #[must_use] pub fn default_private_accounts_for_wallet() -> Vec { - let mut key_chains = vec![KeyChain::new_os_random(), KeyChain::new_os_random()]; + let mut key_chains = DEFAULT_PRIVATE_ACCOUNT_SEEDS + .iter() + .map(|seed| deterministic_private_key_chain(*seed)) + .collect::>(); key_chains.sort_unstable(); key_chains @@ -125,6 +144,25 @@ pub fn default_private_accounts_for_wallet() -> Vec KeyChain { + let mnemonic = + bip39::Mnemonic::from_entropy(&entropy).expect("32 bytes of entropy is valid for bip39"); + let seed_holder = SeedHolder::from_mnemonic(&mnemonic, ""); + + let secret_spending_key = seed_holder.produce_top_secret_key_holder(); + let private_key_holder = secret_spending_key.produce_private_key_holder(None); + let nullifier_public_key = private_key_holder.generate_nullifier_public_key(); + let viewing_public_key = private_key_holder.generate_viewing_public_key(); + + KeyChain { + secret_spending_key, + private_key_holder, + nullifier_public_key, + viewing_public_key, + } +} + #[must_use] pub fn genesis_from_accounts( public_accounts: &[(PrivateKey, u128)], @@ -181,6 +219,7 @@ pub fn indexer_config( channel_id, cross_zone, bridge_lock_holdings: Vec::new(), + allow_chain_reset: false, }) } diff --git a/test_fixtures/src/lib.rs b/test_fixtures/src/lib.rs index 5edc64b3..ff87c8e9 100644 --- a/test_fixtures/src/lib.rs +++ b/test_fixtures/src/lib.rs @@ -16,13 +16,17 @@ use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuil use serde::Serialize; use tempfile::TempDir; use testcontainers::compose::DockerCompose; -use wallet::{WalletCore, account::AccountIdWithPrivacy, cli::CliAccountMention}; +use wallet::{ + WalletCore, account::AccountIdWithPrivacy, cli::CliAccountMention, + config::WalletConfigOverrides, +}; use crate::{ indexer_client::IndexerClient, setup::{ setup_bedrock_node, setup_indexer, setup_private_accounts_with_initial_supply, - setup_public_accounts_with_initial_supply, setup_sequencer, setup_wallet, + setup_public_accounts_with_initial_supply, setup_sequencer, setup_sequencer_from_prebuilt, + setup_wallet, sync_wallet_from_prebuilt, }, }; @@ -92,7 +96,7 @@ impl TestContext { /// Get a builder for the test context to customize its configuration. #[must_use] - pub const fn builder() -> TestContextBuilder { + pub fn builder() -> TestContextBuilder { TestContextBuilder::new() } @@ -250,17 +254,31 @@ pub struct TestContextBuilder { genesis_transactions: Option>, sequencer_partial_config: Option, enable_indexer: bool, + wallet_config_overrides: WalletConfigOverrides, + from_scratch: bool, } impl TestContextBuilder { - const fn new() -> Self { + fn new() -> Self { Self { genesis_transactions: None, sequencer_partial_config: None, enable_indexer: true, + wallet_config_overrides: WalletConfigOverrides::default(), + from_scratch: false, } } + /// Override wallet config fields (e.g. polling timeouts) for the wallet built by this context. + #[must_use] + pub fn with_wallet_config_overrides( + mut self, + wallet_config_overrides: WalletConfigOverrides, + ) -> Self { + self.wallet_config_overrides = wallet_config_overrides; + self + } + #[must_use] pub fn with_genesis(mut self, genesis_transactions: Vec) -> Self { self.genesis_transactions = Some(genesis_transactions); @@ -276,6 +294,14 @@ impl TestContextBuilder { self } + /// Build from genesis live instead of loading the prebuilt fixture. Implied by + /// [`Self::with_genesis`]. + #[must_use] + pub const fn from_scratch(mut self) -> Self { + self.from_scratch = true; + self + } + /// Exclude Indexer from test context. /// Indexer is enabled by default. /// @@ -292,6 +318,8 @@ impl TestContextBuilder { genesis_transactions, sequencer_partial_config, enable_indexer, + wallet_config_overrides, + from_scratch, } = self; // Ensure logger is initialized only once @@ -299,6 +327,10 @@ impl TestContextBuilder { debug!("Test context setup"); + // The fixture bakes in the default accounts + genesis, so custom genesis / from_scratch + // must build live. Otherwise load the fixture (fails if it is missing). + let use_prebuilt = !from_scratch && genesis_transactions.is_none(); + let (bedrock_compose, bedrock_addr) = setup_bedrock_node() .await .context("Failed to setup Bedrock node")?; @@ -324,42 +356,59 @@ impl TestContextBuilder { let initial_public_accounts = config::default_public_accounts_for_wallet(); let initial_private_accounts = config::default_private_accounts_for_wallet(); - // Wallet genesis must always be present so that - // setup_public/private_accounts_with_initial_supply can claim from the vault PDAs. - // When a test supplies custom genesis, merge rather than replace. - let wallet_genesis = - config::genesis_from_accounts(&initial_public_accounts, &initial_private_accounts); - let genesis = match genesis_transactions { - Some(mut custom) => { - custom.extend(wallet_genesis); - custom - } - None => wallet_genesis, + + let partial_config = sequencer_partial_config.unwrap_or_default(); + + let (sequencer_handle, temp_sequencer_dir) = if use_prebuilt { + setup_sequencer_from_prebuilt(partial_config, bedrock_addr) + .await + .context("Failed to setup Sequencer from prebuilt database")? + } else { + // Wallet genesis must always be present so that + // setup_public/private_accounts_with_initial_supply can claim from the vault PDAs. + // When a test supplies custom genesis, merge rather than replace. + let wallet_genesis = + config::genesis_from_accounts(&initial_public_accounts, &initial_private_accounts); + let genesis = match genesis_transactions { + Some(mut custom) => { + custom.extend(wallet_genesis); + custom + } + None => wallet_genesis, + }; + setup_sequencer( + partial_config, + bedrock_addr, + genesis, + config::bedrock_channel_id(), + None, + ) + .await + .context("Failed to setup Sequencer")? }; - let (sequencer_handle, temp_sequencer_dir) = setup_sequencer( - sequencer_partial_config.unwrap_or_default(), - bedrock_addr, - genesis, - config::bedrock_channel_id(), - None, - ) - .await - .context("Failed to setup Sequencer")?; let (mut wallet, temp_wallet_dir, wallet_password) = setup_wallet( sequencer_handle.addr(), &initial_public_accounts, &initial_private_accounts, + wallet_config_overrides, ) .context("Failed to setup wallet")?; - setup_public_accounts_with_initial_supply(&mut wallet, &initial_public_accounts) - .await - .context("Failed to initialize public accounts in wallet")?; + if use_prebuilt { + // Funds already exist on-chain in the prebuilt blocks; sync instead of claiming live. + sync_wallet_from_prebuilt(&mut wallet) + .await + .context("Failed to sync wallet from prebuilt database")?; + } else { + setup_public_accounts_with_initial_supply(&mut wallet, &initial_public_accounts) + .await + .context("Failed to initialize public accounts in wallet")?; - setup_private_accounts_with_initial_supply(&mut wallet, &initial_private_accounts) - .await - .context("Failed to initialize private accounts in wallet")?; + setup_private_accounts_with_initial_supply(&mut wallet, &initial_private_accounts) + .await + .context("Failed to initialize private accounts in wallet")?; + } let sequencer_url = config::addr_to_url(config::UrlProtocol::Http, sequencer_handle.addr()) .context("Failed to convert sequencer addr to URL")?; @@ -391,6 +440,7 @@ impl TestContextBuilder { }) } } + /// A test context to be used in normal #[test] tests. pub struct BlockingTestContext { ctx: Option, @@ -462,7 +512,7 @@ pub async fn fetch_privacy_preserving_tx( seq_client: &SequencerClient, tx_hash: HashType, ) -> PrivacyPreservingTransaction { - let tx = seq_client.get_transaction(tx_hash).await.unwrap().unwrap(); + let (tx, _block_id) = seq_client.get_transaction(tx_hash).await.unwrap().unwrap(); match tx { LeeTransaction::PrivacyPreserving(privacy_preserving_transaction) => { diff --git a/test_fixtures/src/setup.rs b/test_fixtures/src/setup.rs index 27d6a079..325e1628 100644 --- a/test_fixtures/src/setup.rs +++ b/test_fixtures/src/setup.rs @@ -4,6 +4,7 @@ use anyhow::{Context as _, Result, bail}; use indexer_service::{ChannelId, IndexerHandle}; use lee::{AccountId, PrivateKey, PublicKey}; use log::{debug, warn}; +use sequencer_core::block_store::{DbDump, SequencerStore}; use sequencer_service::{GenesisAction, SequencerHandle}; use tempfile::TempDir; use testcontainers::compose::DockerCompose; @@ -19,6 +20,28 @@ use crate::{ private_mention, public_mention, }; +/// How to initialize the sequencer's database. +pub enum SequencerInit<'dump> { + /// Apply these genesis actions from scratch. + Genesis(Vec), + /// Restore an existing chain from a prebuilt dump (genesis actions are then irrelevant). + Prebuilt(&'dump DbDump), +} + +/// Committed single-file dump of the prebuilt sequencer database (`just regenerate-test-fixture`). +#[must_use] +pub fn prebuilt_sequencer_db_dump_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/prebuilt_sequencer_db.dump") +} + +/// Load and deserialize the committed prebuilt-database dump. +fn load_prebuilt_dump() -> Result { + let path = prebuilt_sequencer_db_dump_path(); + let bytes = std::fs::read(&path) + .with_context(|| format!("Failed to read prebuilt db dump at {}", path.display()))?; + DbDump::from_bytes(&bytes).context("Failed to deserialize prebuilt db dump") +} + pub async fn setup_bedrock_node() -> Result<(DockerCompose, SocketAddr)> { let manifest_dir = env!("CARGO_MANIFEST_DIR"); let bedrock_compose_path = PathBuf::from(manifest_dir).join("../bedrock/docker-compose.yml"); @@ -105,10 +128,15 @@ pub async fn setup_indexer( let indexer_config = config::indexer_config(bedrock_addr, channel_id, cross_zone) .context("Failed to create Indexer config")?; - indexer_service::run_server(indexer_config, temp_indexer_dir.path(), 0) - .await - .context("Failed to run Indexer Service") - .map(|handle| (handle, temp_indexer_dir)) + indexer_service::run_server( + indexer_config, + temp_indexer_dir.path(), + 0, + tokio_util::sync::CancellationToken::new(), + ) + .await + .context("Failed to run Indexer Service") + .map(|handle| (handle, temp_indexer_dir)) } pub async fn setup_sequencer( @@ -117,6 +145,40 @@ pub async fn setup_sequencer( genesis_transactions: Vec, channel_id: ChannelId, cross_zone: Option, +) -> Result<(SequencerHandle, TempDir)> { + setup_sequencer_inner( + partial, + bedrock_addr, + SequencerInit::Genesis(genesis_transactions), + channel_id, + cross_zone, + ) + .await +} + +/// Like [`setup_sequencer`], but rebuilds the sequencer database from the committed prebuilt dump +/// so it starts from an existing chain instead of applying genesis from scratch. +pub async fn setup_sequencer_from_prebuilt( + partial: config::SequencerPartialConfig, + bedrock_addr: SocketAddr, +) -> Result<(SequencerHandle, TempDir)> { + let dump = load_prebuilt_dump()?; + setup_sequencer_inner( + partial, + bedrock_addr, + SequencerInit::Prebuilt(&dump), + config::bedrock_channel_id(), + None, + ) + .await +} + +async fn setup_sequencer_inner( + partial: config::SequencerPartialConfig, + bedrock_addr: SocketAddr, + init: SequencerInit<'_>, + channel_id: ChannelId, + cross_zone: Option, ) -> Result<(SequencerHandle, TempDir)> { let temp_sequencer_dir = tempfile::tempdir().context("Failed to create temp dir for sequencer home")?; @@ -126,6 +188,17 @@ pub async fn setup_sequencer( temp_sequencer_dir.path().display() ); + let genesis_transactions = match init { + SequencerInit::Genesis(genesis) => genesis, + SequencerInit::Prebuilt(dump) => { + // `SequencerCore::open_or_create_store` looks for `/rocksdb`. + let dst = temp_sequencer_dir.path().join("rocksdb"); + SequencerStore::restore_db_from_dump(&dst, dump) + .context("Failed to restore prebuilt sequencer database from dump")?; + Vec::new() + } + }; + let config = config::sequencer_config( partial, temp_sequencer_dir.path().to_owned(), @@ -145,6 +218,7 @@ pub fn setup_wallet( sequencer_addr: SocketAddr, initial_public_accounts: &[(PrivateKey, u128)], initial_private_accounts: &[InitialPrivateAccountForWallet], + config_overrides: WalletConfigOverrides, ) -> Result<(WalletCore, TempDir, String)> { let config = config::wallet_config(sequencer_addr).context("Failed to create Wallet config")?; let config_serialized = @@ -158,7 +232,6 @@ pub fn setup_wallet( .context("Failed to write wallet config in temp dir")?; let storage_path = temp_wallet_dir.path().join("storage.json"); - let config_overrides = WalletConfigOverrides::default(); let wallet_password = "test_pass".to_owned(); let (mut wallet, _mnemonic) = WalletCore::new_init_storage( @@ -200,10 +273,13 @@ pub async fn setup_public_accounts_with_initial_supply( initial_public_accounts: &[(PrivateKey, u128)], ) -> Result<()> { for (private_key, amount) in initial_public_accounts { - claim_funds_from_vault( + let account_id = AccountId::from(&PublicKey::new_from_private_key(private_key)); + wallet::cli::execute_subcommand( wallet, - AccountId::from(&PublicKey::new_from_private_key(private_key)), - *amount, + Command::Vault(VaultSubcommand::Claim { + account_id: public_mention(account_id), + amount: *amount, + }), ) .await .context("Failed to claim funds from vault into public account")?; @@ -229,24 +305,11 @@ pub async fn setup_private_accounts_with_initial_supply( Ok(()) } -async fn claim_funds_from_vault( - wallet: &mut WalletCore, - owner_id: AccountId, - amount: u128, -) -> Result<()> { - let result = wallet::cli::execute_subcommand( - wallet, - Command::Vault(VaultSubcommand::Claim { - account_id: public_mention(owner_id), - amount, - }), - ) - .await - .context("Failed to execute public vault claim command")?; - - let SubcommandReturnValue::Empty = result else { - bail!("Expected Empty return value for public vault claim"); - }; +pub async fn sync_wallet_from_prebuilt(wallet: &mut WalletCore) -> Result<()> { + wallet + .sync_to_latest_block() + .await + .context("Failed to sync wallet from prebuilt chain")?; Ok(()) } @@ -270,8 +333,8 @@ async fn claim_funds_from_vault_to_private( .await .context("Failed to execute private vault claim command")?; - let SubcommandReturnValue::PrivacyPreservingTransfer { .. } = result else { - bail!("Expected PrivacyPreservingTransfer return value for private vault claim"); + let SubcommandReturnValue::TransactionExecuted { .. } = result else { + bail!("Expected TransactionExecuted return value for private vault claim"); }; Ok(()) diff --git a/test_fixtures/tests/prebuilt_fixture.rs b/test_fixtures/tests/prebuilt_fixture.rs new file mode 100644 index 00000000..4960af81 --- /dev/null +++ b/test_fixtures/tests/prebuilt_fixture.rs @@ -0,0 +1,58 @@ +//! "Genesis follows config" checks for both the from-scratch and prebuilt `TestContext` paths. +//! Need Docker/Bedrock, like the integration tests. +#![expect(clippy::tests_outside_test_module, reason = "It's a tests module")] + +use anyhow::{Context as _, Result}; +use lee::{AccountId, PublicKey}; +use sequencer_service_rpc::RpcClient as _; +use test_fixtures::{ + TestContext, + config::{default_private_accounts_for_wallet, default_public_accounts_for_wallet}, + verify_commitment_is_in_state, +}; + +/// Builds from genesis (no prebuilt database) and checks the on-chain state follows the config. +#[tokio::test] +async fn genesis_from_scratch_follows_config() -> Result<()> { + let ctx = TestContext::builder().from_scratch().build().await?; + assert_context_follows_config(&ctx).await +} + +/// Loads the prebuilt database via [`TestContext::new`] and checks the restored state follows the +/// config โ€” i.e. the dump round-trips and all expected data persists. +#[tokio::test] +async fn prebuilt_context_follows_config() -> Result<()> { + let ctx = TestContext::new().await?; + assert_context_follows_config(&ctx).await +} + +/// Assert every configured default account is funded with its configured balance, and every +/// configured private account's commitment is present in sequencer state. (The wallet also has an +/// unfunded default root account, so we check the configured accounts specifically.) +async fn assert_context_follows_config(ctx: &TestContext) -> Result<()> { + for (private_key, expected_balance) in default_public_accounts_for_wallet() { + let account_id = AccountId::from(&PublicKey::new_from_private_key(&private_key)); + let balance = ctx + .sequencer_client() + .get_account_balance(account_id) + .await?; + assert_eq!( + balance, expected_balance, + "public account {account_id} balance" + ); + } + + for account in default_private_accounts_for_wallet() { + let account_id = account.account_id(); + let commitment = ctx + .wallet() + .get_private_account_commitment(account_id) + .with_context(|| format!("commitment for private account {account_id}"))?; + assert!( + verify_commitment_is_in_state(commitment, ctx.sequencer_client()).await, + "private commitment for {account_id} should be in sequencer state" + ); + } + + Ok(()) +} diff --git a/tools/crypto_primitives_bench/Cargo.toml b/tools/crypto_primitives_bench/Cargo.toml index 780fac77..30d38681 100644 --- a/tools/crypto_primitives_bench/Cargo.toml +++ b/tools/crypto_primitives_bench/Cargo.toml @@ -11,7 +11,6 @@ workspace = true [dev-dependencies] key_protocol.workspace = true lee_core = { workspace = true, features = ["host"] } -rand = { workspace = true } criterion.workspace = true [[bench]] diff --git a/tools/crypto_primitives_bench/benches/primitives.rs b/tools/crypto_primitives_bench/benches/primitives.rs index 11c11d9b..9a42305c 100644 --- a/tools/crypto_primitives_bench/benches/primitives.rs +++ b/tools/crypto_primitives_bench/benches/primitives.rs @@ -48,7 +48,8 @@ fn bench_encryption(c: &mut Criterion) { let recipient_kc = KeyChain::new_os_random(); let npk = recipient_kc.nullifier_public_key; let account = Account::default(); - let account_id = AccountId::for_regular_private_account(&npk, 0); + let account_id = + AccountId::for_regular_private_account(&npk, &recipient_kc.viewing_public_key, 0); let commitment = Commitment::new(&account_id, &account); let (shared, _epk) = SharedSecretKey::encapsulate(&recipient_kc.viewing_public_key); let kind = PrivateAccountKind::Regular(0_u128); diff --git a/tools/integration_bench/src/harness.rs b/tools/integration_bench/src/harness.rs index 369387e3..37015003 100644 --- a/tools/integration_bench/src/harness.rs +++ b/tools/integration_bench/src/harness.rs @@ -108,7 +108,7 @@ async fn begin_step(ctx: &TestContext) -> Result { /// Finish a timed wallet step. Records submit (the time between `started` /// being captured and `ret` being received) and, if `ret` is a -/// [`SubcommandReturnValue::PrivacyPreservingTransfer`], polls the sequencer +/// [`SubcommandReturnValue::TransactionExecuted`], polls the sequencer /// for inclusion and records the inclusion latency. Returns a [`StepResult`]. async fn finalize_step( label: impl Into, @@ -132,7 +132,7 @@ async fn finalize_step( // recorded" signal. let should_wait_for_chain = !matches!(ret, SubcommandReturnValue::RegisterAccount { .. }); if should_wait_for_chain { - if let SubcommandReturnValue::PrivacyPreservingTransfer { tx_hash } = ret { + if let SubcommandReturnValue::TransactionExecuted { tx_hash } = ret { tx_hash_str = Some(format!("{tx_hash}")); } let started_inclusion = Instant::now();