From 1ae7192c7aaca785ff937fd4484d494d47de0312 Mon Sep 17 00:00:00 2001 From: Pravdyvy Date: Fri, 3 Apr 2026 15:50:24 +0300 Subject: [PATCH] fix: trying to run tests --- Cargo.lock | 13 + Cargo.toml | 3 +- indexer_ffi/Cargo.toml | 2 +- indexer_ffi/src/errors.rs | 2 +- indexer_ffi/src/indexer.rs | 30 +- indexer_ffi/src/lib.rs | 2 +- indexer_service_ffi/Cargo.toml | 17 + indexer_service_ffi/src/main.rs | 41 +++ integration_tests/Cargo.toml | 1 + integration_tests/src/lib.rs | 1 + integration_tests/src/test_context_ffi.rs | 404 ++++++++++++++++++++++ integration_tests/tests/indexer.rs | 34 +- 12 files changed, 537 insertions(+), 13 deletions(-) create mode 100644 indexer_service_ffi/Cargo.toml create mode 100644 indexer_service_ffi/src/main.rs create mode 100644 integration_tests/src/test_context_ffi.rs diff --git a/Cargo.lock b/Cargo.lock index d77d05ef..e5efdd6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3487,6 +3487,18 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "indexer_service_ffi" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "env_logger", + "indexer_ffi", + "log", + "tokio", +] + [[package]] name = "indexer_service_protocol" version = "0.1.0" @@ -3579,6 +3591,7 @@ dependencies = [ "env_logger", "futures", "hex", + "indexer_ffi", "indexer_service", "indexer_service_rpc", "key_protocol", diff --git a/Cargo.toml b/Cargo.toml index e31f9085..4ae2e09b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ members = [ "examples/program_deployment/methods/guest", "bedrock_client", "testnet_initial_state", - "indexer_ffi", + "indexer_ffi", "indexer_service_ffi", ] [workspace.dependencies] @@ -57,6 +57,7 @@ indexer_service_protocol = { path = "indexer/service/protocol" } indexer_service_rpc = { path = "indexer/service/rpc" } wallet = { path = "wallet" } wallet-ffi = { path = "wallet-ffi", default-features = false } +indexer_ffi = { path = "indexer_ffi" } token_core = { path = "programs/token/core" } token_program = { path = "programs/token" } amm_core = { path = "programs/amm/core" } diff --git a/indexer_ffi/Cargo.toml b/indexer_ffi/Cargo.toml index ed2ff4c6..b55230c6 100644 --- a/indexer_ffi/Cargo.toml +++ b/indexer_ffi/Cargo.toml @@ -13,7 +13,7 @@ tokio = { features = ["rt-multi-thread"], workspace = true } cbindgen = "0.29" [lib] -crate-type = ["cdylib"] +crate-type = ["rlib", "cdylib", "staticlib"] name = "indexer_ffi" [lints] diff --git a/indexer_ffi/src/errors.rs b/indexer_ffi/src/errors.rs index 92b61e10..46aa0f9f 100644 --- a/indexer_ffi/src/errors.rs +++ b/indexer_ffi/src/errors.rs @@ -1,4 +1,4 @@ -#[derive(Default, PartialEq, Eq)] +#[derive(Debug, Default, PartialEq, Eq)] #[repr(C)] pub enum OperationStatus { #[default] diff --git a/indexer_ffi/src/indexer.rs b/indexer_ffi/src/indexer.rs index 23f81d2b..0cd0c980 100644 --- a/indexer_ffi/src/indexer.rs +++ b/indexer_ffi/src/indexer.rs @@ -1,4 +1,4 @@ -use std::ffi::c_void; +use std::{ffi::c_void, net::SocketAddr}; use indexer_service::IndexerHandle; use tokio::runtime::Runtime; @@ -21,9 +21,33 @@ impl IndexerServiceFFI { // Helper to safely take ownership back #[must_use] pub fn into_parts(self) -> (Box, Box) { - let overwatch = unsafe { Box::from_raw(self.indexer_handle.cast::()) }; + let indexer_handle = unsafe { Box::from_raw(self.indexer_handle.cast::()) }; let runtime = unsafe { Box::from_raw(self.runtime.cast::()) }; - (overwatch, runtime) + (indexer_handle, runtime) + } + + // Helper to get indexer handle addr + pub unsafe fn addr(&self) -> SocketAddr { + let indexer_handle = unsafe { + self.indexer_handle + .cast::() + .as_ref() + .expect("Indexr Handle must be non-null pointer") + }; + + indexer_handle.addr() + } + + // Helper to get indexer handle addr + pub unsafe fn handle(&self) -> &IndexerHandle { + let indexer_handle = unsafe { + self.indexer_handle + .cast::() + .as_ref() + .expect("Indexr Handle must be non-null pointer") + }; + + indexer_handle } } diff --git a/indexer_ffi/src/lib.rs b/indexer_ffi/src/lib.rs index 289def52..fe594ec0 100644 --- a/indexer_ffi/src/lib.rs +++ b/indexer_ffi/src/lib.rs @@ -3,6 +3,6 @@ pub use errors::OperationStatus; pub use indexer::IndexerServiceFFI; -mod api; +pub mod api; mod errors; mod indexer; diff --git a/indexer_service_ffi/Cargo.toml b/indexer_service_ffi/Cargo.toml new file mode 100644 index 00000000..5d1cfabc --- /dev/null +++ b/indexer_service_ffi/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "indexer_service_ffi" +version = "0.1.0" +edition = "2024" +license.workspace = true + +[dependencies] +indexer_ffi.workspace = true + +log.workspace = true +clap.workspace = true +anyhow.workspace = true +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] } +env_logger.workspace = true + +[lints] +workspace = true diff --git a/indexer_service_ffi/src/main.rs b/indexer_service_ffi/src/main.rs new file mode 100644 index 00000000..e4077e18 --- /dev/null +++ b/indexer_service_ffi/src/main.rs @@ -0,0 +1,41 @@ +use std::{ffi::{CString, c_char}, path::PathBuf}; + +use anyhow::Result; +use clap::Parser; +use indexer_ffi::api::lifecycle::InitializedIndexerServiceFFIResult; +use log::info; + +#[derive(Debug, Parser)] +#[clap(version)] +struct Args { + #[clap(name = "config")] + config_path: PathBuf, + #[clap(short, long, default_value = "8779")] + port: u16, +} + +unsafe extern "C" { + fn start_indexer(config_path: *const c_char, port: u16) -> InitializedIndexerServiceFFIResult; +} + +#[expect( + clippy::integer_division_remainder_used, + reason = "Generated by select! macro, can't be easily rewritten to avoid this lint" +)] +fn main() -> Result<()> { + env_logger::init(); + + let Args { config_path, port } = Args::parse(); + + let res = + unsafe { start_indexer(CString::new(config_path.to_str().unwrap())?.as_ptr(), port) }; + + if res.error.is_error() { + anyhow::bail!("Indexer FFI error {:?}", res.error); + } + + loop { + std::thread::sleep(std::time::Duration::from_secs(10)); + info!("Running..."); + } +} diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index cb5277d2..53f0ee98 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -22,6 +22,7 @@ ata_core.workspace = true indexer_service_rpc.workspace = true sequencer_service_rpc = { workspace = true, features = ["client"] } wallet-ffi.workspace = true +indexer_ffi.workspace = true testnet_initial_state.workspace = true url.workspace = true diff --git a/integration_tests/src/lib.rs b/integration_tests/src/lib.rs index a4381acf..09017db3 100644 --- a/integration_tests/src/lib.rs +++ b/integration_tests/src/lib.rs @@ -17,6 +17,7 @@ use testcontainers::compose::DockerCompose; use wallet::{WalletCore, config::WalletConfigOverrides}; pub mod config; +pub mod test_context_ffi; // TODO: Remove this and control time from tests pub const TIME_TO_WAIT_FOR_BLOCK_SECONDS: u64 = 12; diff --git a/integration_tests/src/test_context_ffi.rs b/integration_tests/src/test_context_ffi.rs new file mode 100644 index 00000000..eb268d6b --- /dev/null +++ b/integration_tests/src/test_context_ffi.rs @@ -0,0 +1,404 @@ +use std::{ + ffi::{CString, c_char}, fs::File, io::Write, net::SocketAddr, path::PathBuf +}; + +use anyhow::{Context, Result, bail}; +use futures::FutureExt; +use indexer_ffi::{IndexerServiceFFI, api::lifecycle::InitializedIndexerServiceFFIResult}; +use log::{debug, error, warn}; +use nssa::AccountId; +use sequencer_core::indexer_client::{IndexerClient, IndexerClientTrait}; +use sequencer_service::SequencerHandle; +use sequencer_service_rpc::{SequencerClient, SequencerClientBuilder}; +use tempfile::TempDir; +use testcontainers::compose::DockerCompose; +use wallet::{WalletCore, config::WalletConfigOverrides}; + +use indexer_service_rpc::RpcClient as _; + +use crate::{ + BEDROCK_SERVICE_PORT, BEDROCK_SERVICE_WITH_OPEN_PORT, LOGGER, TestContextBuilder, config, +}; + +unsafe extern "C" { + fn start_indexer(config_path: *const c_char, port: u16) -> InitializedIndexerServiceFFIResult; +} + +/// Test context which sets up a sequencer, indexer through ffi and a wallet for integration tests. +/// +/// It's memory and logically safe to create multiple instances of this struct in parallel tests, +/// as each instance uses its own temporary directories for sequencer and wallet data. +// NOTE: Order of fields is important for proper drop order. +pub struct TestContextFFI { + sequencer_client: SequencerClient, + indexer_client: IndexerClient, + wallet: WalletCore, + wallet_password: String, + /// Optional to move out value in Drop. + sequencer_handle: Option, + indexer_ffi: IndexerServiceFFI, + bedrock_compose: DockerCompose, + _temp_indexer_dir: TempDir, + _temp_sequencer_dir: TempDir, + _temp_wallet_dir: TempDir, + _runtime: tokio::runtime::Runtime, +} + +impl TestContextBuilder { + pub fn build_ffi(self, runtime: tokio::runtime::Runtime) -> Result { + TestContextFFI::new_configured( + self.sequencer_partial_config.unwrap_or_default(), + self.initial_data.unwrap_or_else(|| { + config::InitialData::with_two_public_and_two_private_initialized_accounts() + }), + runtime, + ) + } +} + +impl TestContextFFI { + /// Create new test context. + pub fn new(runtime: tokio::runtime::Runtime) -> Result { + Self::builder().build_ffi(runtime) + } + + #[must_use] + pub const fn builder() -> TestContextBuilder { + TestContextBuilder::new() + } + + fn new_configured( + sequencer_partial_config: config::SequencerPartialConfig, + initial_data: config::InitialData, + runtime: tokio::runtime::Runtime + ) -> Result { + // Ensure logger is initialized only once + *LOGGER; + + debug!("Test context setup"); + + let (bedrock_compose, bedrock_addr) = runtime.block_on(Self::setup_bedrock_node())?; + + let (indexer_ffi, temp_indexer_dir) = Self::setup_indexer_ffi(bedrock_addr, &initial_data) + .context("Failed to setup Indexer")?; + + let (sequencer_handle, temp_sequencer_dir) = runtime.block_on(Self::setup_sequencer( + sequencer_partial_config, + bedrock_addr, + unsafe { indexer_ffi.addr() }, + &initial_data, + )) + .context("Failed to setup Sequencer")?; + + let (wallet, temp_wallet_dir, wallet_password) = + runtime.block_on(Self::setup_wallet(sequencer_handle.addr(), &initial_data) + ) + .context("Failed to setup wallet")?; + + let sequencer_url = config::addr_to_url(config::UrlProtocol::Http, sequencer_handle.addr()) + .context("Failed to convert sequencer addr to URL")?; + let indexer_url = + config::addr_to_url(config::UrlProtocol::Ws, unsafe { indexer_ffi.addr() }) + .context("Failed to convert indexer addr to URL")?; + let sequencer_client = SequencerClientBuilder::default() + .build(sequencer_url) + .context("Failed to create sequencer client")?; + let indexer_client = runtime.block_on(IndexerClient::new(&indexer_url) +) + .context("Failed to create indexer client")?; + + Ok(Self { + sequencer_client, + indexer_client, + wallet, + wallet_password, + bedrock_compose, + sequencer_handle: Some(sequencer_handle), + indexer_ffi, + _temp_indexer_dir: temp_indexer_dir, + _temp_sequencer_dir: temp_sequencer_dir, + _temp_wallet_dir: temp_wallet_dir, + _runtime: runtime, + }) + } + + async fn setup_bedrock_node() -> Result<(DockerCompose, SocketAddr)> { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + let bedrock_compose_path = + PathBuf::from(manifest_dir).join("../bedrock/docker-compose.yml"); + + let mut compose = DockerCompose::with_auto_client(&[bedrock_compose_path]) + .await + .context("Failed to setup docker compose for Bedrock")? + // Setting port to 0 to avoid conflicts between parallel tests, actual port will be retrieved after container is up + .with_env("PORT", "0"); + + #[expect( + clippy::items_after_statements, + reason = "This is more readable is this function used just after its definition" + )] + async fn up_and_retrieve_port(compose: &mut DockerCompose) -> Result { + compose + .up() + .await + .context("Failed to bring up Bedrock services")?; + let container = compose + .service(BEDROCK_SERVICE_WITH_OPEN_PORT) + .with_context(|| { + format!( + "Failed to get Bedrock service container `{BEDROCK_SERVICE_WITH_OPEN_PORT}`" + ) + })?; + + let ports = container.ports().await.with_context(|| { + format!( + "Failed to get ports for Bedrock service container `{}`", + container.id() + ) + })?; + ports + .map_to_host_port_ipv4(BEDROCK_SERVICE_PORT) + .with_context(|| { + format!( + "Failed to retrieve host port of {BEDROCK_SERVICE_PORT} container \ + port for container `{}`, existing ports: {ports:?}", + container.id() + ) + }) + } + + let mut port = None; + let mut attempt = 0_u32; + let max_attempts = 5_u32; + while port.is_none() && attempt < max_attempts { + attempt = attempt + .checked_add(1) + .expect("We check that attempt < max_attempts, so this won't overflow"); + match up_and_retrieve_port(&mut compose).await { + Ok(p) => { + port = Some(p); + } + Err(err) => { + warn!( + "Failed to bring up Bedrock services: {err:?}, attempt {attempt}/{max_attempts}" + ); + } + } + } + let Some(port) = port else { + bail!("Failed to bring up Bedrock services after {max_attempts} attempts"); + }; + + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + Ok((compose, addr)) + } + + fn setup_indexer_ffi( + bedrock_addr: SocketAddr, + initial_data: &config::InitialData, + ) -> Result<(IndexerServiceFFI, TempDir)> { + let temp_indexer_dir = + tempfile::tempdir().context("Failed to create temp dir for indexer home")?; + + debug!( + "Using temp indexer home at {}", + temp_indexer_dir.path().display() + ); + + let indexer_config = config::indexer_config( + bedrock_addr, + temp_indexer_dir.path().to_owned(), + initial_data, + ) + .context("Failed to create Indexer config")?; + + let config_json = serde_json::to_vec(&indexer_config)?; + let config_path = temp_indexer_dir.path().join("indexer_config.json"); + let mut file = File::create(config_path.as_path())?; + file.write_all(&config_json)?; + file.flush()?; + + let res = + unsafe { start_indexer(CString::new(config_path.to_str().unwrap())?.as_ptr(), 0) }; + + if res.error.is_error() { + anyhow::bail!("Indexer FFI error {:?}", res.error); + } + + Ok((unsafe { std::ptr::read(res.value) }, temp_indexer_dir)) + } + + async fn setup_sequencer( + partial: config::SequencerPartialConfig, + bedrock_addr: SocketAddr, + indexer_addr: SocketAddr, + initial_data: &config::InitialData, + ) -> Result<(SequencerHandle, TempDir)> { + let temp_sequencer_dir = + tempfile::tempdir().context("Failed to create temp dir for sequencer home")?; + + debug!( + "Using temp sequencer home at {}", + temp_sequencer_dir.path().display() + ); + + let config = config::sequencer_config( + partial, + temp_sequencer_dir.path().to_owned(), + bedrock_addr, + indexer_addr, + initial_data, + ) + .context("Failed to create Sequencer config")?; + + let sequencer_handle = sequencer_service::run(config, 0).await?; + + Ok((sequencer_handle, temp_sequencer_dir)) + } + + async fn setup_wallet( + sequencer_addr: SocketAddr, + initial_data: &config::InitialData, + ) -> Result<(WalletCore, TempDir, String)> { + let config = config::wallet_config(sequencer_addr, initial_data) + .context("Failed to create Wallet config")?; + let config_serialized = + serde_json::to_string_pretty(&config).context("Failed to serialize Wallet config")?; + + let temp_wallet_dir = + tempfile::tempdir().context("Failed to create temp dir for wallet home")?; + + let config_path = temp_wallet_dir.path().join("wallet_config.json"); + std::fs::write(&config_path, config_serialized) + .context("Failed to write wallet config in temp dir")?; + + let storage_path = temp_wallet_dir.path().join("storage.json"); + let config_overrides = WalletConfigOverrides::default(); + + let wallet_password = "test_pass".to_owned(); + let (wallet, _mnemonic) = WalletCore::new_init_storage( + config_path, + storage_path, + Some(config_overrides), + &wallet_password, + ) + .context("Failed to init wallet")?; + wallet + .store_persistent_data() + .await + .context("Failed to store wallet persistent data")?; + + Ok((wallet, temp_wallet_dir, wallet_password)) + } + + /// Get reference to the wallet. + #[must_use] + pub const fn wallet(&self) -> &WalletCore { + &self.wallet + } + + #[must_use] + pub fn wallet_password(&self) -> &str { + &self.wallet_password + } + + /// Get mutable reference to the wallet. + pub const fn wallet_mut(&mut self) -> &mut WalletCore { + &mut self.wallet + } + + /// Get reference to the sequencer client. + #[must_use] + pub const fn sequencer_client(&self) -> &SequencerClient { + &self.sequencer_client + } + + /// Get reference to the indexer client. + #[must_use] + pub const fn indexer_client(&self) -> &IndexerClient { + &self.indexer_client + } + + /// Get existing public account IDs in the wallet. + #[must_use] + pub fn existing_public_accounts(&self) -> Vec { + self.wallet + .storage() + .user_data + .public_account_ids() + .collect() + } + + /// Get existing private account IDs in the wallet. + #[must_use] + pub fn existing_private_accounts(&self) -> Vec { + self.wallet + .storage() + .user_data + .private_account_ids() + .collect() + } + + pub fn get_last_block_sequencer(&self) -> Result { + Ok(self._runtime.block_on(self.sequencer_client.get_last_finalized_block_id())?) + } + + pub fn get_last_block_indexer(&self) -> Result { + Ok(self._runtime.block_on(self.indexer_client.get_last_finalized_block_id())?) + } +} + +impl Drop for TestContextFFI { + fn drop(&mut self) { + let Self { + sequencer_handle, + indexer_ffi, + bedrock_compose, + _temp_indexer_dir: _, + _temp_sequencer_dir: _, + _temp_wallet_dir: _, + sequencer_client: _, + indexer_client: _, + wallet: _, + wallet_password: _, + _runtime: _, + } = self; + + let sequencer_handle = sequencer_handle + .take() + .expect("Sequencer handle should be present in TestContext drop"); + if !sequencer_handle.is_healthy() { + let Err(err) = sequencer_handle + .failed() + .now_or_never() + .expect("Sequencer handle should not be running"); + error!( + "Sequencer handle has unexpectedly stopped before TestContext drop with error: {err:#}" + ); + } + + let indexer_handle = unsafe { indexer_ffi.handle() }; + + if !indexer_handle.is_healthy() { + error!("Indexer handle has unexpectedly stopped before TestContext drop"); + } + + let container = bedrock_compose + .service(BEDROCK_SERVICE_WITH_OPEN_PORT) + .unwrap_or_else(|| { + panic!("Failed to get Bedrock service container `{BEDROCK_SERVICE_WITH_OPEN_PORT}`") + }); + let output = std::process::Command::new("docker") + .args(["inspect", "-f", "{{.State.Running}}", container.id()]) + .output() + .expect("Failed to execute docker inspect command to check if Bedrock container is still running"); + let stdout = String::from_utf8(output.stdout) + .expect("Failed to parse docker inspect output as String"); + if stdout.trim() != "true" { + error!( + "Bedrock container `{}` is not running during TestContext drop, docker inspect output: {stdout}", + container.id() + ); + } + } +} diff --git a/integration_tests/tests/indexer.rs b/integration_tests/tests/indexer.rs index cb8cf0e9..738a3364 100644 --- a/integration_tests/tests/indexer.rs +++ b/integration_tests/tests/indexer.rs @@ -7,15 +7,14 @@ use std::time::Duration; use anyhow::Result; use indexer_service_rpc::RpcClient as _; -use integration_tests::{TIME_TO_WAIT_FOR_BLOCK_SECONDS, TestContext, format_public_account_id}; +use integration_tests::{TIME_TO_WAIT_FOR_BLOCK_SECONDS, TestContext, format_public_account_id, test_context_ffi::TestContextFFI}; use log::info; -use tokio::test; use wallet::cli::{Command, programs::native_token_transfer::AuthTransferSubcommand}; /// Timeout in milliseconds to reliably await for block finalization. -const L2_TO_L1_TIMEOUT_MILLIS: u64 = 600_000; +const L2_TO_L1_TIMEOUT_MILLIS: u64 = 100_000; -#[test] +#[tokio::test] async fn indexer_test_run() -> Result<()> { let ctx = TestContext::new().await?; @@ -40,7 +39,7 @@ async fn indexer_test_run() -> Result<()> { Ok(()) } -#[test] +#[tokio::test] async fn indexer_block_batching() -> Result<()> { let ctx = TestContext::new().await?; @@ -78,7 +77,7 @@ async fn indexer_block_batching() -> Result<()> { Ok(()) } -#[test] +#[tokio::test] async fn indexer_state_consistency() -> Result<()> { let mut ctx = TestContext::new().await?; @@ -147,3 +146,26 @@ async fn indexer_state_consistency() -> Result<()> { Ok(()) } + +#[test] +fn indexer_test_run_ffi() -> Result<()> { + println!("Hello 1"); + let runtime = tokio::runtime::Runtime::new()?; + println!("Hello 2"); + let _ctx = TestContextFFI::new(runtime)?; + + log::info!("Hello 3"); + + // RUN OBSERVATION + std::thread::sleep(std::time::Duration::from_millis(L2_TO_L1_TIMEOUT_MILLIS)); + + let last_block_seq = _ctx.get_last_block_sequencer()?; + let last_block_indexer = _ctx.get_last_block_indexer()?; + + println!("Last block on ind now is {last_block_indexer}"); + println!("Last block on seq now is {last_block_seq}"); + + assert!(last_block_indexer > 1); + + Ok(()) +}