diff --git a/.gitignore b/.gitignore index 8b98abd..82b175b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ Cargo.lock target/ -.vscode \ No newline at end of file +.vscode +.idea \ No newline at end of file diff --git a/surrealdb_bench/Cargo.toml b/surrealdb_bench/Cargo.toml new file mode 100644 index 0000000..3a4edd2 --- /dev/null +++ b/surrealdb_bench/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "surrealdb_bench" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.98" +clap = { version = "4.5.37", features = ["derive", "help", "color"] } +rocksdb = "0.23.0" +serde = { version = "1.0.219", features = ["derive"] } +serde_json = "1.0.140" +surrealdb = { version = "2.2.2", features = ["kv-rocksdb", "kv-mem", "allocator"] } +tempfile = "3.19.1" +tokio = { version = "1.44.2", features = ["macros", "rt-multi-thread", "sync"] } +regex = "1.11.1" + +[profile.release] +lto = true +strip = true +opt-level = 3 +panic = 'abort' +codegen-units = 1 + +[[bin]] +name = "surrealdb_select_all" +path = "src/bin/surrealdb_select_all.rs" + +[[bin]] +name = "surrealdb_parent_to_child_graph" +path = "src/bin/surrealdb_parent_to_child_graph.rs" + +[[bin]] +name = "surrealdb_parent_to_child_pointers" +path = "src/bin/surrealdb_parent_to_child_pointers.rs" + +[[bin]] +name = "rocksdb_select_all" +path = "src/bin/rocksdb_select_all.rs" + +[[bin]] +name = "rocksdb_parent_to_child" +path = "src/bin/rocksdb_parent_child.rs" \ No newline at end of file diff --git a/surrealdb_bench/README.md b/surrealdb_bench/README.md new file mode 100644 index 0000000..b67cae3 --- /dev/null +++ b/surrealdb_bench/README.md @@ -0,0 +1,33 @@ +# SurrealDB Benchmarks + +This project contains a set of benches to benchmark and compare the performance of SurrealDB and RocksDB. + +## Benches + +### SurrealDB Binaries + +1. **`surrealdb_select_all`** + - Selecting all records from a SurrealDB database. + +2. **`surrealdb_parent_to_child_graph`** + - Measures the performance of querying child-to-parent relationships using SurrealDB graph model(child->parent edges) + +3. **`surrealdb_parent_to_child_pointers`** + - Measures the performance of querying child-to-parent relationships by querying parents by their key + +### RocksDB Binaries + +1. **`rocksdb_select_all`** + - Benchmarks the performance of selecting all records + +2. **`rocksdb_parent_to_child`** + - Measures the performance of querying child-to-parent relationships by querying parents by their key + +## Usage +To run the benchmarks, you can use the following command: + +```bash + ./run_benchmarks.sh [BLOCKS_COUNT] +``` + +Default `BLOCKS_COUNT` is 1000000 \ No newline at end of file diff --git a/surrealdb_bench/run_benchmarks.sh b/surrealdb_bench/run_benchmarks.sh new file mode 100755 index 0000000..e9ecfad --- /dev/null +++ b/surrealdb_bench/run_benchmarks.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +BUILD_MODE="release" + +BIN_DIR="target/$BUILD_MODE" + +BENCHES=( + "surrealdb_select_all" + "surrealdb_parent_to_child_graph" + "surrealdb_parent_to_child_pointers" + "rocksdb_select_all" + "rocksdb_parent_to_child" +) + +# Read blocks_count from the command-line argument or default to 1000000 +BLOCKS_COUNT=${1:-1000000} + +echo "Building all binaries..." +cargo build --$BUILD_MODE +if [ $? -ne 0 ]; then + echo "Error: Build failed." + exit 1 +fi + + +echo "Running benchmarks with blocks_count=$BLOCKS_COUNT" +for BIN in "${BENCHES[@]}"; do + "$BIN_DIR/$BIN" --blocks-count "$BLOCKS_COUNT" + if [ $? -ne 0 ]; then + echo "Error: $BIN failed to execute." + exit 1 + fi +done + +echo "All benchmarks executed successfully." \ No newline at end of file diff --git a/surrealdb_bench/src/bin/rocksdb_parent_child.rs b/surrealdb_bench/src/bin/rocksdb_parent_child.rs new file mode 100644 index 0000000..345684e --- /dev/null +++ b/surrealdb_bench/src/bin/rocksdb_parent_child.rs @@ -0,0 +1,66 @@ +use anyhow::Error; +use clap::Parser; +use rocksdb::DB; +use serde_json::{self, from_slice}; +use std::time::Instant; +use surrealdb_bench::common::{Args, Block}; +use surrealdb_bench::rocksdb::{setup_blocks, setup_db}; +use tempfile::TempDir; + +fn main() -> Result<(), Error> { + println!("----------------------------------------"); + println!("[BENCHMARK START] ROCKSDB PARENT-TO-CHILD"); + + let args = Args::parse(); + let temp_dir = TempDir::new()?; + let db = setup_db(&temp_dir)?; + + let start_setup = Instant::now(); + setup_blocks(&db, args.blocks_count)?; + + let duration_setup = start_setup.elapsed(); + println!( + "[BENCHMARK SETUP] {} BLOCKS TOOK {:?}", + args.blocks_count, duration_setup + ); + + let start = Instant::now(); + let target_hash = format!("block{}", args.blocks_count - 1); + + let ancestors = get_ancestors(&db, &target_hash, args.blocks_count)?; + assert_eq!(ancestors.len(), args.blocks_count); + + let duration = start.elapsed(); + println!( + "[BENCHMARK RESULT] PARENT-TO-CHILD TRAVERSAL ({} BLOCKS) TOOK: {:?}", + args.blocks_count, duration + ); + + println!("----------------------------------------\n"); + Ok(()) +} + +/// Fetch a block from the database by its hash. +fn get_block(db: &DB, hash: &str) -> Result, Error> { + let key = hash.as_bytes(); + Ok(match db.get(key)? { + Some(value) => Some(from_slice(&value)?), + None => None, + }) +} + +/// Retrieve all ancestors of a block by following the parent pointers. +fn get_ancestors(db: &DB, hash: &str, count: usize) -> Result, Error> { + let mut ancestors = Vec::with_capacity(count); + let mut current_hash = hash.to_string(); + + while !current_hash.is_empty() { + if let Some(block) = get_block(db, ¤t_hash)? { + current_hash = block.parent.clone(); + ancestors.push(block); + } else { + break; + } + } + Ok(ancestors) +} diff --git a/surrealdb_bench/src/bin/rocksdb_select_all.rs b/surrealdb_bench/src/bin/rocksdb_select_all.rs new file mode 100644 index 0000000..1fa9547 --- /dev/null +++ b/surrealdb_bench/src/bin/rocksdb_select_all.rs @@ -0,0 +1,52 @@ +use anyhow::Error; +use clap::Parser; +use rocksdb::DB; +use serde_json::{self, from_slice}; +use std::time::Instant; +use surrealdb_bench::common::{Args, Block}; +use surrealdb_bench::rocksdb::{setup_blocks, setup_db}; +use tempfile::TempDir; + +#[tokio::main] +async fn main() -> Result<(), Error> { + println!("----------------------------------------"); + println!("[BENCHMARK START] ROCKSDB SELECT ALL"); + + let args = Args::parse(); + let temp_dir = TempDir::new()?; + let db = setup_db(&temp_dir)?; + + let start_setup = Instant::now(); + setup_blocks(&db, args.blocks_count)?; + + let setup_duration = start_setup.elapsed(); + println!( + "[BENCHMARK SETUP] {} BLOCKS TOOK {:?}", + args.blocks_count, setup_duration + ); + + let start_query = Instant::now(); + + let blocks = fetch_all_blocks(&db, args.blocks_count)?; + assert_eq!(blocks.len(), args.blocks_count); + + let query_duration = start_query.elapsed(); + println!( + "[BENCHMARK RESULT] SELECTING {} BLOCKS TOOK {:?}", + args.blocks_count, query_duration + ); + + println!("----------------------------------------\n"); + Ok(()) +} + +fn fetch_all_blocks(db: &DB, count: usize) -> Result, Error> { + let mut blocks = Vec::with_capacity(count); + let iter = db.iterator(rocksdb::IteratorMode::Start); + for item in iter { + let (_, value) = item?; + let block: Block = from_slice(&value)?; + blocks.push(block); + } + Ok(blocks) +} diff --git a/surrealdb_bench/src/bin/surrealdb_parent_to_child_graph.rs b/surrealdb_bench/src/bin/surrealdb_parent_to_child_graph.rs new file mode 100644 index 0000000..82b6b51 --- /dev/null +++ b/surrealdb_bench/src/bin/surrealdb_parent_to_child_graph.rs @@ -0,0 +1,58 @@ +use anyhow::Error; +use clap::Parser; +use serde::Deserialize; +use std::time::Instant; +use surrealdb::Surreal; +use surrealdb::engine::local::Db; +use surrealdb_bench::common::{Args, Block}; +use surrealdb_bench::surrealdb::{Relation, insert_blocks, insert_edges, setup_db}; +use tempfile::TempDir; + +#[derive(Debug, Deserialize)] +struct ParentBlock { + #[serde(default)] + parent: Vec, +} + +#[tokio::main] +async fn main() -> Result<(), Error> { + println!("----------------------------------------"); + println!("[BENCHMARK START] SURREALDB PARENT-TO-CHILD GRAPH"); + + let args = Args::parse(); + let temp_dir = TempDir::new()?; + let db = setup_db(&temp_dir).await?; + + let setup_duration = Instant::now(); + let blocks = insert_blocks(&db, args.blocks_count).await?; + insert_edges(&db, blocks, Relation::Parent).await?; + + let setup_duration = setup_duration.elapsed(); + println!( + "[BENCHMARK SETUP] {} BLOCKS (INCLUDING EDGES) TOOK {:?}", + args.blocks_count, setup_duration + ); + + let start_query = Instant::now(); + let parent_blocks = fetch_parent_blocks(&db).await?; + let parent_blocks: Vec = parent_blocks.into_iter().flat_map(|pb| pb.parent).collect(); + + assert_eq!(parent_blocks.len(), args.blocks_count - 1); + + let query_duration = start_query.elapsed(); + println!( + "[BENCHMARK RESULT] PARENT-TO-CHILD TRAVERSAL ({} BLOCKS) TOOK {:?}", + args.blocks_count, query_duration + ); + + println!("----------------------------------------\n"); + Ok(()) +} + +/// Fetches parent blocks from the database by following the parent relation. +async fn fetch_parent_blocks(db: &Surreal) -> Result, Error> { + let query = "SELECT ->parent->block.* AS parent FROM block"; + let mut response = db.query(query).await?.check()?; + let blocks: Vec = response.take(0)?; + Ok(blocks) +} diff --git a/surrealdb_bench/src/bin/surrealdb_parent_to_child_pointers.rs b/surrealdb_bench/src/bin/surrealdb_parent_to_child_pointers.rs new file mode 100644 index 0000000..507c010 --- /dev/null +++ b/surrealdb_bench/src/bin/surrealdb_parent_to_child_pointers.rs @@ -0,0 +1,64 @@ +use anyhow::Error; +use clap::Parser; +use std::time::Instant; +use surrealdb::Surreal; +use surrealdb::engine::local::Db; +use surrealdb_bench::common::{Args, Block}; +use surrealdb_bench::surrealdb::{insert_blocks, setup_db}; +use tempfile::TempDir; + +#[tokio::main] +async fn main() -> Result<(), Error> { + println!("----------------------------------------"); + println!("[BENCHMARK START] SURREALDB PARENT-TO-CHILD POINTER"); + + let args = Args::parse(); + let temp_dir = TempDir::new()?; + let db = setup_db(&temp_dir).await?; + + let start_setup = Instant::now(); + insert_blocks(&db, args.blocks_count).await?; + + let setup_duration = start_setup.elapsed(); + println!( + "[BENCHMARK SETUP] {} BLOCKS (WITHOUT EDGES) TOOK {:?}", + args.blocks_count, setup_duration + ); + + let start_query = Instant::now(); + + let target_hash = format!("block{}", args.blocks_count - 1); + + let ancestors = get_ancestors(&db, &target_hash, args.blocks_count).await?; + assert_eq!(ancestors.len(), args.blocks_count); + + let query_duration = start_query.elapsed(); + println!( + "[BENCHMARK RESULT] PARENT-TO-CHILD TRAVERSAL ({} BLOCKS) TOOK {:?}", + args.blocks_count, query_duration + ); + + println!("----------------------------------------\n"); + Ok(()) +} + +/// Select a specific block by its hash from the database. +async fn get_block(db: &Surreal, hash: &str) -> Result, Error> { + db.select(("block", hash)).await.map_err(Into::into) +} + +/// Retrieve all blocks by following the parent pointers from a given block hash. +async fn get_ancestors(db: &Surreal, hash: &str, count: usize) -> Result, Error> { + let mut ancestors = Vec::with_capacity(count); + let mut current_hash = hash.to_string(); + + while !current_hash.is_empty() { + if let Some(block) = get_block(db, ¤t_hash).await? { + ancestors.push(block.clone()); + current_hash = block.parent; + } else { + break; + } + } + Ok(ancestors) +} diff --git a/surrealdb_bench/src/bin/surrealdb_select_all.rs b/surrealdb_bench/src/bin/surrealdb_select_all.rs new file mode 100644 index 0000000..954ade7 --- /dev/null +++ b/surrealdb_bench/src/bin/surrealdb_select_all.rs @@ -0,0 +1,49 @@ +use anyhow::Error; +use clap::Parser; +use std::time::Instant; +use surrealdb::Surreal; +use surrealdb::engine::local::Db; +use surrealdb_bench::common::{Args, Block}; +use surrealdb_bench::surrealdb::{insert_blocks, setup_db}; +use tempfile::TempDir; + +#[tokio::main] +async fn main() -> Result<(), Error> { + println!("----------------------------------------"); + println!("[BENCHMARK START] SURREALDB SELECT ALL"); + + let args = Args::parse(); + let temp_dir = TempDir::new()?; + let db = setup_db(&temp_dir).await?; + + let start_setup = Instant::now(); + insert_blocks(&db, args.blocks_count).await?; + + let start_setup = start_setup.elapsed(); + println!( + "[BENCHMARK SETUP] {} BLOCKS TOOK {:?}", + args.blocks_count, start_setup + ); + + let start_query = Instant::now(); + + let blocks = fetch_all_blocks(&db).await?; + assert_eq!(blocks.len(), args.blocks_count,); + + let query_duration = start_query.elapsed(); + println!( + "[BENCHMARK RESULT] SELECTING {} BLOCKS TOOK {:?}", + args.blocks_count, query_duration + ); + + println!("----------------------------------------\n"); + Ok(()) +} + +/// Fetches all blocks from the database. +async fn fetch_all_blocks(db: &Surreal) -> Result, Error> { + let query = "SELECT * FROM block"; + let mut response = db.query(query).await?; + let blocks: Vec = response.take(0)?; + Ok(blocks) +} diff --git a/surrealdb_bench/src/common.rs b/surrealdb_bench/src/common.rs new file mode 100644 index 0000000..6ba9dc7 --- /dev/null +++ b/surrealdb_bench/src/common.rs @@ -0,0 +1,15 @@ +use clap::Parser; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Block { + pub hash: String, + pub parent: String, + pub height: u64, +} + +#[derive(Parser)] +pub struct Args { + #[arg(long, default_value_t = 1_000_000)] + pub blocks_count: usize, +} \ No newline at end of file diff --git a/surrealdb_bench/src/lib.rs b/surrealdb_bench/src/lib.rs new file mode 100644 index 0000000..80ad77a --- /dev/null +++ b/surrealdb_bench/src/lib.rs @@ -0,0 +1,3 @@ +pub mod common; +pub mod rocksdb; +pub mod surrealdb; diff --git a/surrealdb_bench/src/rocksdb.rs b/surrealdb_bench/src/rocksdb.rs new file mode 100644 index 0000000..47fd71b --- /dev/null +++ b/surrealdb_bench/src/rocksdb.rs @@ -0,0 +1,21 @@ +use crate::surrealdb::create_n_blocks; +use anyhow::Error; +use rocksdb::DB; +use serde_json::to_vec; +use tempfile::TempDir; + +pub fn setup_db(temp_dir: &TempDir) -> Result { + Ok(DB::open_default(temp_dir.path())?) +} + +pub fn setup_blocks(db: &DB, count: usize) -> Result<(), Error> { + let blocks = create_n_blocks(count); + let mut batch = rocksdb::WriteBatch::default(); + for block in &blocks { + let key = block.hash.as_bytes(); + let value = to_vec(block)?; + batch.put(key, value); + } + db.write(batch)?; + Ok(()) +} diff --git a/surrealdb_bench/src/surrealdb.rs b/surrealdb_bench/src/surrealdb.rs new file mode 100644 index 0000000..52e647d --- /dev/null +++ b/surrealdb_bench/src/surrealdb.rs @@ -0,0 +1,80 @@ +use crate::common::Block; +use anyhow::Error; +use serde::Deserialize; +use surrealdb::Surreal; +use surrealdb::engine::local::{Db, RocksDb}; +use tempfile::TempDir; + +#[derive(Debug, Deserialize)] +pub enum Relation { + Parent, + Child, +} + +pub async fn setup_db(temp_dir: &TempDir) -> Result, Error> { + let db_path = temp_dir.path().join("surreal.db"); + let db = Surreal::new::(db_path).await?; + db.use_ns("nomos").use_db("nms").await?; + Ok(db) +} + +pub async fn insert_blocks(db: &Surreal, count: usize) -> Result, Error> { + let blocks = create_n_blocks(count); + let queries: Vec = blocks + .iter() + .map(|block| { + Ok(format!( + "CREATE block:{} CONTENT {};", + block.hash, + serde_json::to_string(block)? + )) + }) + .collect::, Error>>()?; + + db.query(queries.join(" ")).await?.check()?; + Ok(blocks) +} + +pub async fn insert_edges( + db: &Surreal, + blocks: Vec, + relation: Relation, +) -> Result<(), Error> { + let blocks = match relation { + Relation::Parent => blocks.into_iter().rev().collect::>(), + Relation::Child => blocks, + }; + + let queries: Vec = blocks + .iter() + .filter(|block| !block.parent.is_empty()) + .map(|block| match relation { + Relation::Parent => format!( + "RELATE block:{}->parent->block:{}", + block.hash, block.parent + ), + Relation::Child => { + format!("RELATE block:{}->child->block:{}", block.parent, block.hash) + } + }) + .collect(); + + if !queries.is_empty() { + db.query(queries.join("; ")).await?.check()?; + } + Ok(()) +} + +pub fn create_n_blocks(n: usize) -> Vec { + (0..n) + .map(|i| Block { + hash: format!("block{}", i), + parent: if i == 0 { + String::new() + } else { + format!("block{}", i - 1) + }, + height: i as u64 + 1, + }) + .collect() +}