diff --git a/.gitignore b/.gitignore index 787fae8..e18a6c8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ *$py.class *.so simulation +network-runner/target diff --git a/network-runner/Cargo.lock b/network-runner/Cargo.lock new file mode 100644 index 0000000..058e1c5 --- /dev/null +++ b/network-runner/Cargo.lock @@ -0,0 +1,2835 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + +[[package]] +name = "anyhow" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" + +[[package]] +name = "array-init-cursor" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" + +[[package]] +name = "arrow-format" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07884ea216994cdc32a2d5f8274a8bee979cfe90274b83f86f440866ee3132c7" +dependencies = [ + "planus", + "serde", +] + +[[package]] +name = "arrow2" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a4c5b03335bc1cb0fd9f5297f8fd3bbfd6fb04f3cb0bc7d6c91b7128cb8336a" +dependencies = [ + "ahash", + "arrow-format", + "base64", + "bytemuck", + "chrono", + "dyn-clone", + "either", + "ethnum", + "fallible-streaming-iterator", + "foreign_vec", + "futures", + "getrandom", + "hash_hasher", + "indexmap", + "json-deserializer", + "lexical-core", + "lz4", + "multiversion", + "num-traits", + "parquet2", + "rustc_version", + "simdutf8", + "streaming-iterator", + "strength_reduce", + "zstd 0.11.2+zstd.1.5.2", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bls-signatures" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edceb821f01a1c93593fbbe47786e01c69c3141baa8e349366f15472b9252bc9" +dependencies = [ + "bls12_381", + "ff", + "group", + "hkdf", + "pairing", + "rand_core", + "rayon", + "sha2", + "subtle", + "thiserror", +] + +[[package]] +name = "bls12_381" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc6d6292be3a19e6379786dac800f551e5865a5bb51ebbe3064ab80433f403" +dependencies = [ + "digest 0.9.0", + "ff", + "group", + "pairing", + "rand_core", + "subtle", +] + +[[package]] +name = "brotli" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "bytemuck" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" + +[[package]] +name = "cc" +version = "1.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.6", +] + +[[package]] +name = "clap" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "comfy-table" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e959d788268e3bf9d35ace83e81b124190378e4c91c9067524675e33394b8ba" +dependencies = [ + "crossterm", + "strum", + "strum_macros", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crossterm" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +dependencies = [ + "bitflags 1.3.2", + "crossterm_winapi", + "libc", + "mio", + "parking_lot 0.12.3", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctrlc" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" +dependencies = [ + "nix", + "windows-sys 0.59.0", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.87", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dyn-clone" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "ethnum" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "ff" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" +dependencies = [ + "bitvec", + "rand_core", + "subtle", +] + +[[package]] +name = "fixed-slice-deque" +version = "0.1.0-beta2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae2a0cf5db6443a1723e95f7798b97ff9f3e7e9dd789caee6e1e320ee2adf2a" +dependencies = [ + "slice-deque", +] + +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "halfbrown" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2a3c70a9c00cc1ee87b54e89f9505f73bb17d63f1b25c9a462ba8ef885444f" +dependencies = [ + "fxhash", + "hashbrown 0.13.2", + "serde", +] + +[[package]] +name = "hash_hasher" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", + "rayon", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01706d578d5c281058480e673ae4086a9f4710d8df1ad80a5b03e39ece5f886b" +dependencies = [ + "digest 0.9.0", + "hmac", +] + +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac", + "digest 0.9.0", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-deserializer" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f63b421e16eb4100beb677af56f0b4f3a4f08bab74ef2af079ce5bb92c2683f" +dependencies = [ + "indexmap", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6" +dependencies = [ + "lexical-core", +] + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.161" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", +] + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "lz4" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d1febb2b4a79ddd1980eede06a8f7902197960aa0383ffcfdd62fe723036725" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "multiversion" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373" +dependencies = [ + "multiversion-macros", +] + +[[package]] +name = "multiversion-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nomos-simulations-network-runner" +version = "0.1.0" +dependencies = [ + "anyhow", + "blake2", + "bls-signatures", + "chrono", + "clap", + "crc32fast", + "crossbeam", + "csv", + "ctrlc", + "digest 0.10.7", + "fixed-slice-deque", + "futures", + "getrandom", + "humantime", + "humantime-serde", + "once_cell", + "parking_lot 0.12.3", + "polars", + "rand", + "rayon", + "scopeguard", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "now" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89e9874397a1f0a52fc1f197a8effd9735223cb2390e9dcc83ac6cd02923d0" +dependencies = [ + "chrono", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "pairing" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fec4625e73cf41ef4bb6846cafa6d44736525f442ba45e407c4a000a13996f" +dependencies = [ + "group", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.10", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.7", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "parquet-format-safe" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1131c54b167dd4e4799ce762e1ab01549ebb94d5bdd13e6ec1b467491c378e1f" +dependencies = [ + "async-trait", + "futures", +] + +[[package]] +name = "parquet2" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579fe5745f02cef3d5f236bfed216fd4693e49e4e920a13475c6132233283bce" +dependencies = [ + "async-stream", + "brotli", + "flate2", + "futures", + "lz4", + "parquet-format-safe", + "seq-macro", + "snap", + "streaming-decompression", + "zstd 0.12.4", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "planus" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1691dd09e82f428ce8d6310bd6d5da2557c82ff17694d2a32cad7242aea89f" +dependencies = [ + "array-init-cursor", +] + +[[package]] +name = "polars" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8918f4add49e6244bae2fe91cac89be339f2d6a77c59a4df3d7348bd40f98d1e" +dependencies = [ + "getrandom", + "polars-core", + "polars-io", + "polars-lazy", + "polars-ops", + "polars-time", + "version_check", +] + +[[package]] +name = "polars-arrow" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e57a7b929edf6c73475dbc3f63d35152f14f4a9455476acc6127d770daa0f6" +dependencies = [ + "arrow2", + "hashbrown 0.13.2", + "num", + "thiserror", +] + +[[package]] +name = "polars-core" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a440cd53916f1a87fac1fda36cd7cc2d226247b4d4570d96242da5fa7f07b2a" +dependencies = [ + "ahash", + "anyhow", + "arrow2", + "bitflags 1.3.2", + "chrono", + "comfy-table", + "hashbrown 0.13.2", + "indexmap", + "num", + "once_cell", + "polars-arrow", + "polars-utils", + "rand", + "rand_distr", + "rayon", + "regex", + "serde", + "serde_json", + "smartstring", + "thiserror", + "wasm-timer", + "xxhash-rust", +] + +[[package]] +name = "polars-io" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d941750cba70a3acf150b959fcf446c09e8a8a4a35d03472d941bd6740bc43" +dependencies = [ + "ahash", + "anyhow", + "arrow2", + "bytes", + "chrono", + "dirs", + "lexical", + "lexical-core", + "memchr", + "memmap2", + "num", + "once_cell", + "polars-arrow", + "polars-core", + "polars-time", + "polars-utils", + "rayon", + "regex", + "serde_json", + "simd-json", + "simdutf8", +] + +[[package]] +name = "polars-lazy" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d227fcb817485be462748d3172d2e55c61d56fbdc7fd56c24b72fa2e510e7be6" +dependencies = [ + "ahash", + "bitflags 1.3.2", + "glob", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-pipe", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", +] + +[[package]] +name = "polars-ops" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36638340fd9f4377dab11f307877ebb5bdac3bc9b25ea32a771584de76e5280a" +dependencies = [ + "arrow2", + "polars-arrow", + "polars-core", + "polars-utils", +] + +[[package]] +name = "polars-pipe" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac1dfae18a14a812119d3328dae8790be53f1bc0fbcb977606dcf0f181490a8f" +dependencies = [ + "crossbeam-channel", + "enum_dispatch", + "hashbrown 0.13.2", + "num", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-plan", + "polars-utils", + "rayon", + "smartstring", +] + +[[package]] +name = "polars-plan" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca57df4974f25fa0642ae18ef00c0836c5120d4371be77a34d4684173b043c3" +dependencies = [ + "ahash", + "once_cell", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-time", + "polars-utils", + "rayon", +] + +[[package]] +name = "polars-time" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d961a9ccbe3c739de063fbf78449b3c172baf3379f958769c42ee9c309289786" +dependencies = [ + "chrono", + "lexical", + "now", + "once_cell", + "polars-arrow", + "polars-core", + "polars-ops", + "polars-utils", + "regex", +] + +[[package]] +name = "polars-utils" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a741a3325c544c97c7a9ff57d857f089b60041bd92b06c41582df6940ffaa05b" +dependencies = [ + "once_cell", + "rayon", + "sysinfo", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.5", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "seq-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" + +[[package]] +name = "serde" +version = "1.0.214" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.214" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "serde_json" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "simd-json" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3375b6c3d8c048ba09c8b4b6c3f1d3f35e06b71db07d231c323943a949e1b8" +dependencies = [ + "halfbrown", + "lexical-core", + "serde", + "serde_json", + "simdutf8", + "value-trait", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "slice-deque" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31ef6ee280cdefba6d2d0b4b78a84a1c1a3f3a4cec98c2d4231c8bc225de0f25" +dependencies = [ + "libc", + "mach", + "winapi", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "streaming-decompression" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" +dependencies = [ + "fallible-streaming-iterator", +] + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.27.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a902e9050fca0a5d6877550b769abd2bd1ce8c04634b941dbe2809735e1a1e33" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "thiserror" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "value-trait" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "995de1aa349a0dc50f4aa40870dce12961a30229027230bad09acd2843edbe9e" +dependencies = [ + "float-cmp", + "halfbrown", + "itoa", + "ryu", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.87", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +dependencies = [ + "zstd-safe 6.0.6", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "6.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/network-runner/Cargo.toml b/network-runner/Cargo.toml new file mode 100644 index 0000000..9046e0c --- /dev/null +++ b/network-runner/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "nomos-simulations-network-runner" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "simulation" +path = "src/bin/app/main.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1" +blake2 = "0.10" +bls-signatures = "0.14" +digest = "0.10" +csv = "1" +clap = { version = "4", features = ["derive"] } +ctrlc = "3.4" +chrono = { version = "0.4", features = ["serde"] } +crc32fast = "1.3" +crossbeam = { version = "0.8.2", features = ["crossbeam-channel"] } +fixed-slice-deque = "0.1.0-beta2" +futures = "0.3" +humantime = "2.1" +humantime-serde = "1" +once_cell = "1.17" +parking_lot = "0.12" +polars = { version = "0.27", features = ["serde", "object", "json", "csv-file", "parquet", "dtype-struct"], optional = true } +rand = { version = "0.8", features = ["small_rng"] } +rayon = "1.7" +scopeguard = "1" +serde = { version = "1.0", features = ["derive", "rc"] } +serde_with = "2.3" +serde_json = "1.0" +thiserror = "1" +tracing = { version = "0.1", default-features = false, features = ["log", "attributes"] } +tracing-subscriber = { version = "0.3", features = ["json", "env-filter", "tracing-log"]} + +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.2", features = ["js"] } + +[features] +polars = ["dep:polars"] diff --git a/network-runner/config/carnot.json b/network-runner/config/carnot.json new file mode 100644 index 0000000..83bd6fd --- /dev/null +++ b/network-runner/config/carnot.json @@ -0,0 +1,52 @@ +{ + "network_settings": { + "network_behaviors": { + "north america:north america": "50ms", + "north america:europe": "100ms", + "north america:asia": "120ms", + "europe:europe": "50ms", + "europe:asia": "100ms", + "europe:north america": "120ms", + "asia:north america": "100ms", + "asia:europe": "120ms", + "asia:asia": "40ms" + }, + "regions": { + "north america": 0.4, + "europe": 0.4, + "asia": 0.3 + } + }, + "overlay_settings": { + "number_of_committees": 3 + }, + "node_settings": { + "timeout": "1000ms" + }, + "step_time": "10ms", + "runner_settings": "Sync", + "stream_settings": { + "path": "test.csv" + }, + "node_count": 3000, + "views_count": 3, + "leaders_count": 1, + "seed": 0, + "wards": [ + {"max_view": 1} + ], + "record_settings": { + "node_id": true, + "current_view": true, + "highest_voted_view": true, + "local_high_qc": true, + "safe_blocks": true, + "last_view_timeout_qc": true, + "latest_committed_block": true, + "latest_committed_view": true, + "root_committee": true, + "parent_committee": true, + "child_committees": true, + "committed_blocks": true + } +} diff --git a/network-runner/config/carnot_dev.json b/network-runner/config/carnot_dev.json new file mode 100644 index 0000000..b98ced2 --- /dev/null +++ b/network-runner/config/carnot_dev.json @@ -0,0 +1,62 @@ +{ + "network_settings": { + "network_behaviors": { + "north america:north america": "10ms", + "north america:europe": "150ms", + "north america:asia": "250ms", + "europe:europe": "10ms", + "europe:asia": "200ms", + "europe:north america": "150ms", + "asia:north america": "250ms", + "asia:europe": "200ms", + "asia:asia": "10ms" + }, + "regions": { + "north america": 0.4, + "europe": 0.3, + "asia": 0.3 + } + }, + "overlay_settings": { + "number_of_committees": 7 + }, + "node_settings": { + "network_capacity_kbps": 10000024, + "timeout": "10000ms" + }, + "step_time": "100ms", + "runner_settings": "Sync", + "stream_settings": { + "path": "tree_500_7_view_1_default.csv", + "format": "csv" + }, + "node_count": 500, + "views_count": 10, + "leaders_count": 1, + "seed": 0, + "wards": [ + { + "max_view": 1 + }, + { + "stalled_view": { + "consecutive_viewed_checkpoint": null, + "criterion": 0, + "threshold": 100 + } + } + ], + "record_settings": { + "current_view": true, + "highest_voted_view": true, + "local_high_qc": true, + "safe_blocks": false, + "last_view_timeout_qc": true, + "latest_committed_block": true, + "latest_committed_view": true, + "root_committee": false, + "parent_committee": false, + "child_committees": false, + "committed_blocks": false + } +} \ No newline at end of file diff --git a/network-runner/src/bin/app/log.rs b/network-runner/src/bin/app/log.rs new file mode 100644 index 0000000..996f4da --- /dev/null +++ b/network-runner/src/bin/app/log.rs @@ -0,0 +1,86 @@ +use std::{ + fs::File, + io::{stderr, stdout}, + path::PathBuf, + str::FromStr, +}; +use tracing_subscriber::fmt::{format::Format, FormatEvent, FormatFields, SubscriberBuilder}; + +#[derive(Default, Copy, Clone)] +pub enum LogFormat { + #[default] + Plain, + Json, +} + +impl FromStr for LogFormat { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "json" => Ok(LogFormat::Json), + "plain" => Ok(LogFormat::Plain), + _ => Err(anyhow::anyhow!("Unknown log format")), + } + } +} + +#[derive(Default, Clone)] +pub enum LogOutput { + #[default] + StdOut, + StdErr, + File(PathBuf), +} + +impl FromStr for LogOutput { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "stdout" => Ok(Self::StdOut), + "stderr" => Ok(Self::StdErr), + path => Ok(Self::File(PathBuf::from(path))), + } + } +} + +pub fn config_tracing(fmt: LogFormat, file: &LogOutput) { + let filter = std::env::var("SIMULATION_LOG").unwrap_or_else(|_| "info".to_owned()); + + let subscriber = tracing_subscriber::fmt::fmt() + .without_time() + .with_line_number(true) + .with_env_filter(filter) + .with_file(false) + .with_target(true); + + if let LogFormat::Json = fmt { + set_global(subscriber.json(), file); + } else { + set_global(subscriber, file); + } +} + +fn set_global( + subscriber: SubscriberBuilder, tracing_subscriber::EnvFilter>, + output: &LogOutput, +) where + N: for<'writer> FormatFields<'writer> + 'static + Send + Sync, + Format: FormatEvent, + L: Send + Sync + 'static, + T: Send + Sync + 'static, +{ + use tracing::subscriber::set_global_default; + match output { + LogOutput::StdOut => set_global_default(subscriber.with_writer(stdout).finish()), + LogOutput::StdErr => set_global_default(subscriber.with_writer(stderr).finish()), + LogOutput::File(path) => set_global_default( + subscriber + .with_ansi(false) + .with_writer(File::create(path).expect("Unable to create log file")) + .finish(), + ), + } + .expect("Unable to set global default subscriber") +} diff --git a/network-runner/src/bin/app/main.rs b/network-runner/src/bin/app/main.rs new file mode 100644 index 0000000..5c07488 --- /dev/null +++ b/network-runner/src/bin/app/main.rs @@ -0,0 +1,185 @@ +// std +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +// crates +use anyhow::Ok; +use clap::Parser; +use nomos_simulations_network_runner::network::behaviour::create_behaviours; +use nomos_simulations_network_runner::network::regions::{create_regions, RegionsData}; +use nomos_simulations_network_runner::network::Network; +use nomos_simulations_network_runner::node::NodeId; +use nomos_simulations_network_runner::output_processors::{OutData, Record}; +use nomos_simulations_network_runner::runner::{BoxedNode, SimulationRunnerHandle}; +#[cfg(feature = "polars")] +use nomos_simulations_network_runner::streaming::polars::PolarsSubscriber; +use nomos_simulations_network_runner::streaming::{ + io::IOSubscriber, naive::NaiveSubscriber, StreamType, +}; +use parking_lot::Mutex; +use rand::rngs::SmallRng; +use rand::seq::SliceRandom; +use rand::SeedableRng; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; +use serde::de::DeserializeOwned; +use serde::Serialize; +// internal +use nomos_simulations_network_runner::{runner::SimulationRunner, settings::SimulationSettings}; +mod log; + +/// Main simulation wrapper +/// Pipes together the cli arguments with the execution +#[derive(Parser)] +pub struct SimulationApp { + /// Json file path, on `SimulationSettings` format + #[clap(long, short)] + input_settings: PathBuf, + #[clap(long)] + stream_type: Option, + #[clap(long, default_value = "plain")] + log_format: log::LogFormat, + #[clap(long, default_value = "stdout")] + log_to: log::LogOutput, + #[clap(long)] + dump_overlay_info: bool, + #[clap(long)] + no_netcap: bool, +} + +impl SimulationApp { + pub fn run(self) -> anyhow::Result<()> { + let Self { + input_settings, + stream_type, + log_format: _, + log_to: _, + dump_overlay_info, + no_netcap, + } = self; + let simulation_settings: SimulationSettings = load_json_from_file(&input_settings)?; + + let seed = simulation_settings.seed.unwrap_or_else(|| { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs() + }); + let mut rng = SmallRng::seed_from_u64(seed); + let mut node_ids: Vec = (0..simulation_settings.node_count) + .map(|_| todo!()) + .collect(); + node_ids.shuffle(&mut rng); + + let regions = create_regions(&node_ids, &mut rng, &simulation_settings.network_settings); + let behaviours = create_behaviours(&simulation_settings.network_settings); + let regions_data = RegionsData::new(regions, behaviours); + + let ids = node_ids.clone(); + let network = Arc::new(Mutex::new(Network::<()>::new(regions_data, seed))); + + // if dump_overlay_info { + // dump_json_to_file( + // Path::new("overlay_info.json"), + // &overlay_node::overlay_info( + // node_ids.clone(), + // node_ids.first().copied().unwrap(), + // &simulation_settings.overlay_settings, + // ), + // )?; + // } + + // let nodes: Vec> = node_ids + // .par_iter() + // .copied() + // .map(|node_id| todo!()) + // .collect(); + // let network = Arc::try_unwrap(network) + // .expect("network is not used anywhere else") + // .into_inner(); + // run::<_, _, _>(network, nodes, simulation_settings, stream_type)?; + Ok(()) + } +} + +fn run( + network: Network, + nodes: Vec>, + settings: SimulationSettings, + stream_type: Option, +) -> anyhow::Result<()> +where + M: Clone + Send + Sync + 'static, + S: 'static, + T: Serialize + Clone + 'static, +{ + let stream_settings = settings.stream_settings.clone(); + let runner = + SimulationRunner::<_, OutData, S, T>::new(network, nodes, Default::default(), settings)?; + + let handle = match stream_type { + Some(StreamType::Naive) => { + let settings = stream_settings.unwrap_naive(); + runner.simulate_and_subscribe::>(settings)? + } + Some(StreamType::IO) => { + let settings = stream_settings.unwrap_io(); + runner.simulate_and_subscribe::>(settings)? + } + #[cfg(feature = "polars")] + Some(StreamType::Polars) => { + let settings = stream_settings.unwrap_polars(); + runner.simulate_and_subscribe::>(settings)? + } + None => runner.simulate()?, + }; + + signal(handle) +} + +fn signal(handle: SimulationRunnerHandle) -> anyhow::Result<()> { + let handle = Arc::new(handle); + let (tx, rx) = crossbeam::channel::bounded(1); + ctrlc::set_handler(move || { + tx.send(()).unwrap(); + })?; + loop { + crossbeam::select! { + recv(rx) -> _ => { + handle.stop()?; + tracing::info!("gracefully shutdown the simulation app"); + break; + }, + default => { + if handle.is_finished() { + handle.shutdown()?; + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + } + } + Ok(()) +} + +/// Generically load a json file +fn load_json_from_file(path: &Path) -> anyhow::Result { + let f = File::open(path).map_err(Box::new)?; + Ok(serde_json::from_reader(f)?) +} + +fn dump_json_to_file(path: &Path, data: &T) -> anyhow::Result<()> { + let f = File::create(path).map_err(Box::new)?; + Ok(serde_json::to_writer(f, data)?) +} + +fn main() -> anyhow::Result<()> { + let app: SimulationApp = SimulationApp::parse(); + log::config_tracing(app.log_format, &app.log_to); + + if let Err(e) = app.run() { + tracing::error!("error: {}", e); + std::process::exit(1); + } + Ok(()) +} diff --git a/network-runner/src/lib.rs b/network-runner/src/lib.rs new file mode 100644 index 0000000..af34cb3 --- /dev/null +++ b/network-runner/src/lib.rs @@ -0,0 +1,10 @@ +pub mod network; +pub mod node; +pub mod output_processors; +pub mod runner; +pub mod settings; +pub mod streaming; +pub mod warding; + +static START_TIME: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(std::time::Instant::now); diff --git a/network-runner/src/network/behaviour.rs b/network-runner/src/network/behaviour.rs new file mode 100644 index 0000000..20faad1 --- /dev/null +++ b/network-runner/src/network/behaviour.rs @@ -0,0 +1,40 @@ +// std +use std::{collections::HashMap, time::Duration}; +// crates +use rand::Rng; +use serde::{Deserialize, Serialize}; + +use super::{NetworkBehaviourKey, NetworkSettings}; +// internal + +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct NetworkBehaviour { + pub delay: Duration, + pub drop: f64, +} + +impl NetworkBehaviour { + pub fn new(delay: Duration, drop: f64) -> Self { + Self { delay, drop } + } + + pub fn delay(&self) -> Duration { + self.delay + } + + pub fn should_drop(&self, rng: &mut R) -> bool { + rng.gen_bool(self.drop) + } +} + +// Takes a reference to the simulation_settings and returns a HashMap representing the +// network behaviors for pairs of NodeIds. +pub fn create_behaviours( + network_settings: &NetworkSettings, +) -> HashMap { + network_settings + .network_behaviors + .iter() + .map(|(k, d)| (*k, NetworkBehaviour::new(*d, 0.0))) + .collect() +} diff --git a/network-runner/src/network/mod.rs b/network-runner/src/network/mod.rs new file mode 100644 index 0000000..70c413c --- /dev/null +++ b/network-runner/src/network/mod.rs @@ -0,0 +1,814 @@ +// std +use std::{ + collections::HashMap, + ops::Add, + str::FromStr, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +// crates +use crossbeam::channel::{self, Receiver, Sender}; +use parking_lot::Mutex; +use rand::{rngs::SmallRng, Rng, SeedableRng}; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; +// internal +use crate::node::NodeId; + +pub mod behaviour; +pub mod regions; + +type NetworkTime = Instant; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct NetworkBehaviourKey { + pub from: regions::Region, + pub to: regions::Region, +} + +impl NetworkBehaviourKey { + pub fn new(from: regions::Region, to: regions::Region) -> Self { + Self { from, to } + } +} + +impl Serialize for NetworkBehaviourKey { + fn serialize(&self, serializer: S) -> Result { + let s = format!("{}:{}", self.from, self.to); + serializer.serialize_str(&s) + } +} + +impl<'de> Deserialize<'de> for NetworkBehaviourKey { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + let mut split = s.split(':'); + let from = split.next().ok_or(serde::de::Error::custom( + "NetworkBehaviourKey should be in the form of `from_region:to_region`", + ))?; + let to = split.next().ok_or(serde::de::Error::custom( + "NetworkBehaviourKey should be in the form of `from_region:to_region`", + ))?; + Ok(Self::new( + regions::Region::from_str(from).map_err(serde::de::Error::custom)?, + regions::Region::from_str(to).map_err(serde::de::Error::custom)?, + )) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct NetworkSettings { + #[serde(with = "network_behaviors_serde")] + pub network_behaviors: HashMap, + /// Represents node distribution in the simulated regions. + /// The sum of distributions should be 1. + pub regions: HashMap, +} + +/// Ser/Deser `HashMap` to humantime format. +mod network_behaviors_serde { + use super::{Deserialize, Duration, HashMap, NetworkBehaviourKey}; + + /// Have to implement this manually because of the `serde_json` will panic if the key of map + /// is not a string. + pub fn serialize( + vals: &HashMap, + serializer: S, + ) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeMap; + let mut ser = serializer.serialize_map(Some(vals.len()))?; + for (k, v) in vals { + ser.serialize_key(&k)?; + ser.serialize_value(&humantime::format_duration(*v).to_string())?; + } + ser.end() + } + + pub fn deserialize<'de, D>( + deserializer: D, + ) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + let map = HashMap::::deserialize(deserializer)?; + map.into_iter() + .map(|(k, v)| { + let v = humantime::parse_duration(&v).map_err(serde::de::Error::custom)?; + Ok((k, v)) + }) + .collect::, _>>() + } +} + +/// Represents node network capacity and current load in bytes. +#[derive(Debug)] +struct NodeNetworkCapacity { + capacity_bps: Option, + current_load: Mutex, + load_to_flush: AtomicU32, +} + +impl NodeNetworkCapacity { + fn new(capacity_bps: Option) -> Self { + Self { + capacity_bps, + current_load: Mutex::new(0), + load_to_flush: AtomicU32::new(0), + } + } + + fn increase_load(&self, load: u32) -> bool { + if let Some(capacity_bps) = self.capacity_bps { + let mut current_load = self.current_load.lock(); + if *current_load + load <= capacity_bps { + *current_load += load; + true + } else { + false + } + } else { + true + } + } + + fn decrease_load(&self, load: u32) { + self.load_to_flush.fetch_add(load, Ordering::Relaxed); + } + + fn flush_load(&self) { + if self.capacity_bps.is_none() { + return; + } + + let mut s = self.current_load.lock(); + *s -= self.load_to_flush.load(Ordering::Relaxed); + self.load_to_flush.store(0, Ordering::Relaxed); + } +} + +#[derive(Debug)] +pub struct Network { + pub regions: regions::RegionsData, + network_time: NetworkTime, + messages: Vec<(NetworkTime, NetworkMessage)>, + node_network_capacity: HashMap, + from_node_receivers: HashMap>>, + from_node_broadcast_receivers: HashMap>>, + to_node_senders: HashMap>>, + seed: u64, +} + +impl Network +where + M: std::fmt::Debug + Send + Sync + Clone, +{ + pub fn new(regions: regions::RegionsData, seed: u64) -> Self { + Self { + regions, + network_time: Instant::now(), + messages: Vec::new(), + node_network_capacity: HashMap::new(), + from_node_receivers: HashMap::new(), + from_node_broadcast_receivers: HashMap::new(), + to_node_senders: HashMap::new(), + seed, + } + } + + fn send_message_cost( + &self, + rng: &mut R, + node_a: NodeId, + node_b: NodeId, + ) -> Option { + let network_behaviour = self.regions.network_behaviour(node_a, node_b); + (!network_behaviour.should_drop(rng)) + // TODO: use a delay range + .then(|| network_behaviour.delay()) + } + + pub fn connect( + &mut self, + node_id: NodeId, + capacity_bps: Option, + node_message_receiver: Receiver>, + node_message_broadcast_receiver: Receiver>, + ) -> Receiver> { + self.node_network_capacity + .insert(node_id, NodeNetworkCapacity::new(capacity_bps)); + let (to_node_sender, from_network_receiver) = channel::unbounded(); + self.from_node_receivers + .insert(node_id, node_message_receiver); + self.from_node_broadcast_receivers + .insert(node_id, node_message_broadcast_receiver); + self.to_node_senders.insert(node_id, to_node_sender); + from_network_receiver + } + + /// Collects and dispatches messages to connected interfaces. + pub fn step(&mut self, time_passed: Duration) { + self.collect_messages(); + self.dispatch_after(time_passed); + } + + /// Receive and store all messages from nodes. + pub fn collect_messages(&mut self) { + let mut adhoc_messages = self + .from_node_receivers + .par_iter() + .flat_map(|(_, from_node)| { + from_node + .try_iter() + .map(|msg| (self.network_time, msg)) + .collect::>() + }) + .collect(); + self.messages.append(&mut adhoc_messages); + + let mut broadcast_messages = self + .from_node_broadcast_receivers + .iter() + .flat_map(|(_, from_node)| { + from_node.try_iter().flat_map(|msg| { + self.to_node_senders.keys().map(move |recipient| { + let mut m = msg.clone(); + m.to = Some(*recipient); + m + }) + }) + }) + .map(|m| (self.network_time, m)) + .collect::>(); + self.messages.append(&mut broadcast_messages); + } + + /// Reiterate all messages and send to appropriate nodes if simulated + /// delay has passed. + pub fn dispatch_after(&mut self, time_passed: Duration) { + self.network_time += time_passed; + + let delayed = self + .messages + .par_iter() + .filter(|(network_time, message)| { + let mut rng = SmallRng::seed_from_u64(self.seed); + self.send_or_drop_message(&mut rng, network_time, message) + }) + .cloned() + .collect(); + + for (_, c) in self.node_network_capacity.iter() { + c.flush_load(); + } + + self.messages = delayed; + } + + /// Returns true if message needs to be delayed and be dispatched in future. + fn send_or_drop_message( + &self, + rng: &mut R, + network_time: &NetworkTime, + message: &NetworkMessage, + ) -> bool { + let to = message.to.expect("adhoc message has recipient"); + if let Some(delay) = self.send_message_cost(rng, message.from, to) { + let node_capacity = self.node_network_capacity.get(&to).unwrap(); + let should_send = network_time.add(delay) <= self.network_time; + let remaining_size = message.remaining_size(); + if should_send && node_capacity.increase_load(remaining_size) { + let to_node = self.to_node_senders.get(&to).unwrap(); + to_node + .send(message.clone()) + .expect("node should have connection"); + node_capacity.decrease_load(remaining_size); + return false; + } else { + // if we do not need to delay, then we should check if the msg is too large + // if so, we mock the partial sending message behavior + if should_send { + // if remaining is 0, we should send without delay + return self.try_partial_send(node_capacity, message, &to) != 0; + } + return true; + } + } + false + } + + /// Try to apply partial send logic, returns the remaining size of the message + fn try_partial_send( + &self, + node_capacity: &NodeNetworkCapacity, + message: &NetworkMessage, + to: &NodeId, + ) -> u32 { + if let Some(capacity_bps) = node_capacity.capacity_bps { + let mut cap = node_capacity.current_load.lock(); + let sent = capacity_bps - *cap; + *cap = capacity_bps; + let remaining = message.partial_send(sent); + // Message is partially sent, the node capacity needs to be flushed at the end of step even + // if the whole message is not sent. + node_capacity.decrease_load(sent); + if remaining == 0 { + let to_node = self.to_node_senders.get(to).unwrap(); + to_node + .send(message.clone()) + .expect("node should have connection"); + } + remaining + } else { + 0 + } + } +} + +#[derive(Clone, Debug)] +pub struct NetworkMessage { + pub from: NodeId, + pub to: Option, + pub payload: M, + pub remaining: Arc, +} + +impl NetworkMessage { + pub fn new(from: NodeId, to: Option, payload: M, size_bytes: u32) -> Self { + Self { + from, + to, + payload, + remaining: Arc::new(AtomicU32::new(size_bytes)), + } + } + + pub fn payload(&self) -> &M { + &self.payload + } + + pub fn into_payload(self) -> M { + self.payload + } + + fn remaining_size(&self) -> u32 { + self.remaining.load(Ordering::SeqCst) + } + + /// Mock the partial sending of a message behavior, returning the remaining message size. + fn partial_send(&self, size: u32) -> u32 { + self.remaining + .fetch_sub(size, Ordering::SeqCst) + .saturating_sub(size) + } +} + +pub trait PayloadSize { + fn size_bytes(&self) -> u32; +} + +pub trait NetworkInterface { + type Payload; + + fn broadcast(&self, message: Self::Payload); + fn send_message(&self, address: NodeId, message: Self::Payload); + fn receive_messages(&self) -> Vec>; +} + +pub struct InMemoryNetworkInterface { + id: NodeId, + broadcast: Sender>, + sender: Sender>, + receiver: Receiver>, +} + +impl InMemoryNetworkInterface { + pub fn new( + id: NodeId, + broadcast: Sender>, + sender: Sender>, + receiver: Receiver>, + ) -> Self { + Self { + id, + broadcast, + sender, + receiver, + } + } +} + +impl NetworkInterface for InMemoryNetworkInterface { + type Payload = M; + + fn broadcast(&self, message: Self::Payload) { + let size = message.size_bytes(); + let message = NetworkMessage::new(self.id, None, message, size); + self.broadcast.send(message).unwrap(); + } + + fn send_message(&self, address: NodeId, message: Self::Payload) { + let size = message.size_bytes(); + let message = NetworkMessage::new(self.id, Some(address), message, size); + self.sender.send(message).unwrap(); + } + + fn receive_messages(&self) -> Vec> { + self.receiver.try_iter().collect() + } +} + +#[cfg(test)] +mod tests { + use super::{ + behaviour::NetworkBehaviour, + regions::{Region, RegionsData}, + Network, NetworkInterface, NetworkMessage, + }; + use crate::{ + network::NetworkBehaviourKey, + node::{NodeId, NodeIdExt}, + }; + use crossbeam::channel::{self, Receiver, Sender}; + use std::{collections::HashMap, time::Duration}; + + struct MockNetworkInterface { + id: NodeId, + broadcast: Sender>, + sender: Sender>, + receiver: Receiver>, + message_size: u32, + } + + impl MockNetworkInterface { + pub fn new( + id: NodeId, + broadcast: Sender>, + sender: Sender>, + receiver: Receiver>, + message_size: u32, + ) -> Self { + Self { + id, + broadcast, + sender, + receiver, + message_size, + } + } + } + + impl NetworkInterface for MockNetworkInterface { + type Payload = (); + + fn broadcast(&self, message: Self::Payload) { + let message = NetworkMessage::new(self.id, None, message, self.message_size); + self.broadcast.send(message).unwrap(); + } + + fn send_message(&self, address: NodeId, message: Self::Payload) { + let message = NetworkMessage::new(self.id, Some(address), message, self.message_size); + self.sender.send(message).unwrap(); + } + + fn receive_messages(&self) -> Vec> { + self.receiver.try_iter().collect() + } + } + + #[test] + fn send_receive_messages() { + let node_a = NodeId::from_index(0); + let node_b = NodeId::from_index(1); + + let regions = HashMap::from([(Region::Europe, vec![node_a, node_b])]); + let behaviour = HashMap::from([( + NetworkBehaviourKey::new(Region::Europe, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + )]); + let regions_data = RegionsData::new(regions, behaviour); + let mut network = Network::new(regions_data, 0); + + let (from_a_sender, from_a_receiver) = channel::unbounded(); + let (from_a_broadcast_sender, from_a_broadcast_receiver) = channel::unbounded(); + let to_a_receiver = + network.connect(node_a, Some(3), from_a_receiver, from_a_broadcast_receiver); + let a = MockNetworkInterface::new( + node_a, + from_a_broadcast_sender, + from_a_sender, + to_a_receiver, + 1, + ); + + let (from_b_sender, from_b_receiver) = channel::unbounded(); + let (from_b_broadcast_sender, from_b_broadcast_receiver) = channel::unbounded(); + let to_b_receiver = + network.connect(node_b, Some(3), from_b_receiver, from_b_broadcast_receiver); + let b = MockNetworkInterface::new( + node_b, + from_b_broadcast_sender, + from_b_sender, + to_b_receiver, + 1, + ); + + a.send_message(node_b, ()); + network.collect_messages(); + + assert_eq!(a.receive_messages().len(), 0); + assert_eq!(b.receive_messages().len(), 0); + + network.step(Duration::from_millis(0)); + assert_eq!(a.receive_messages().len(), 0); + assert_eq!(b.receive_messages().len(), 0); + + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 0); + assert_eq!(b.receive_messages().len(), 1); + + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 0); + assert_eq!(b.receive_messages().len(), 0); + + b.send_message(node_a, ()); + b.send_message(node_a, ()); + b.send_message(node_a, ()); + network.collect_messages(); + + network.dispatch_after(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 3); + assert_eq!(b.receive_messages().len(), 0); + } + + #[test] + fn regions_send_receive_messages() { + let node_a = NodeId::from_index(0); + let node_b = NodeId::from_index(1); + let node_c = NodeId::from_index(2); + + let regions = HashMap::from([ + (Region::Asia, vec![node_a, node_b]), + (Region::Europe, vec![node_c]), + ]); + let behaviour = HashMap::from([ + ( + NetworkBehaviourKey::new(Region::Asia, Region::Asia), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + ), + ( + NetworkBehaviourKey::new(Region::Asia, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(500), 0.0), + ), + ( + NetworkBehaviourKey::new(Region::Europe, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + ), + ]); + let regions_data = RegionsData::new(regions, behaviour); + let mut network = Network::new(regions_data, 0); + + let (from_a_sender, from_a_receiver) = channel::unbounded(); + let (from_a_broadcast_sender, from_a_broadcast_receiver) = channel::unbounded(); + let to_a_receiver = + network.connect(node_a, Some(2), from_a_receiver, from_a_broadcast_receiver); + let a = MockNetworkInterface::new( + node_a, + from_a_broadcast_sender, + from_a_sender, + to_a_receiver, + 1, + ); + + let (from_b_sender, from_b_receiver) = channel::unbounded(); + let (from_b_broadcast_sender, from_b_broadcast_receiver) = channel::unbounded(); + let to_b_receiver = + network.connect(node_b, Some(2), from_b_receiver, from_b_broadcast_receiver); + let b = MockNetworkInterface::new( + node_b, + from_b_broadcast_sender, + from_b_sender, + to_b_receiver, + 1, + ); + + let (from_c_sender, from_c_receiver) = channel::unbounded(); + let (from_c_broadcast_sender, from_c_broadcast_receiver) = channel::unbounded(); + let to_c_receiver = + network.connect(node_c, Some(2), from_c_receiver, from_c_broadcast_receiver); + let c = MockNetworkInterface::new( + node_c, + from_c_broadcast_sender, + from_c_sender, + to_c_receiver, + 1, + ); + + a.send_message(node_b, ()); + a.send_message(node_c, ()); + network.collect_messages(); + + b.send_message(node_a, ()); + b.send_message(node_c, ()); + network.collect_messages(); + + c.send_message(node_a, ()); + c.send_message(node_b, ()); + network.collect_messages(); + + network.dispatch_after(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 1); + assert_eq!(b.receive_messages().len(), 1); + assert_eq!(c.receive_messages().len(), 0); + + a.send_message(node_b, ()); + b.send_message(node_c, ()); + network.collect_messages(); + + network.dispatch_after(Duration::from_millis(400)); + assert_eq!(a.receive_messages().len(), 1); // c to a + assert_eq!(b.receive_messages().len(), 2); // c to b && a to b + assert_eq!(c.receive_messages().len(), 2); // a to c && b to c + + network.dispatch_after(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 0); + assert_eq!(b.receive_messages().len(), 0); + assert_eq!(c.receive_messages().len(), 1); // b to c + } + + #[test] + fn node_network_capacity_limit() { + let node_a = NodeId::from_index(0); + let node_b = NodeId::from_index(1); + + let regions = HashMap::from([(Region::Europe, vec![node_a, node_b])]); + let behaviour = HashMap::from([( + NetworkBehaviourKey::new(Region::Europe, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + )]); + let regions_data = RegionsData::new(regions, behaviour); + let mut network = Network::new(regions_data, 0); + + let (from_a_sender, from_a_receiver) = channel::unbounded(); + let (from_a_broadcast_sender, from_a_broadcast_receiver) = channel::unbounded(); + let to_a_receiver = + network.connect(node_a, Some(3), from_a_receiver, from_a_broadcast_receiver); + let a = MockNetworkInterface::new( + node_a, + from_a_broadcast_sender, + from_a_sender, + to_a_receiver, + 1, + ); + + let (from_b_sender, from_b_receiver) = channel::unbounded(); + let (from_b_broadcast_sender, from_b_broadcast_receiver) = channel::unbounded(); + let to_b_receiver = + network.connect(node_b, Some(2), from_b_receiver, from_b_broadcast_receiver); + let b = MockNetworkInterface::new( + node_b, + from_b_broadcast_sender, + from_b_sender, + to_b_receiver, + 1, + ); + + for _ in 0..6 { + a.send_message(node_b, ()); + b.send_message(node_a, ()); + } + + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 3); + assert_eq!(b.receive_messages().len(), 2); + + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 3); + assert_eq!(b.receive_messages().len(), 2); + + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 0); + assert_eq!(b.receive_messages().len(), 2); + } + + #[test] + fn node_network_capacity_no_limit() { + let node_a = NodeId::from_index(0); + let node_b = NodeId::from_index(1); + + let regions = HashMap::from([(Region::Europe, vec![node_a, node_b])]); + let behaviour = HashMap::from([( + NetworkBehaviourKey::new(Region::Europe, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + )]); + let regions_data = RegionsData::new(regions, behaviour); + let mut network = Network::new(regions_data, 0); + + let (from_a_sender, from_a_receiver) = channel::unbounded(); + let (from_a_broadcast_sender, from_a_broadcast_receiver) = channel::unbounded(); + let to_a_receiver = + network.connect(node_a, None, from_a_receiver, from_a_broadcast_receiver); + let a = MockNetworkInterface::new( + node_a, + from_a_broadcast_sender, + from_a_sender, + to_a_receiver, + 1000, + ); + + let (from_b_sender, from_b_receiver) = channel::unbounded(); + let (from_b_broadcast_sender, from_b_broadcast_receiver) = channel::unbounded(); + let to_b_receiver = + network.connect(node_b, None, from_b_receiver, from_b_broadcast_receiver); + let b = MockNetworkInterface::new( + node_b, + from_b_broadcast_sender, + from_b_sender, + to_b_receiver, + 100, + ); + + for _ in 0..6 { + a.send_message(node_b, ()); + b.send_message(node_a, ()); + } + + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 6); + assert_eq!(b.receive_messages().len(), 6); + } + + #[test] + fn node_network_message_partial_send() { + let node_a = NodeId::from_index(0); + let node_b = NodeId::from_index(1); + + let regions = HashMap::from([(Region::Europe, vec![node_a, node_b])]); + let behaviour = HashMap::from([( + NetworkBehaviourKey::new(Region::Europe, Region::Europe), + NetworkBehaviour::new(Duration::from_millis(100), 0.0), + )]); + let regions_data = RegionsData::new(regions, behaviour); + let mut network = Network::new(regions_data, 0); + + let (from_a_sender, from_a_receiver) = channel::unbounded(); + let (from_a_broadcast_sender, from_a_broadcast_receiver) = channel::unbounded(); + + // Node A is connected to the network with throuput of 5. + let to_a_receiver = + network.connect(node_a, Some(5), from_a_receiver, from_a_broadcast_receiver); + + // Every message sent **from** Node A will be of size 15. + let a = MockNetworkInterface::new( + node_a, + from_a_broadcast_sender, + from_a_sender, + to_a_receiver, + 2, + ); + + let (from_b_sender, from_b_receiver) = channel::unbounded(); + let (from_b_broadcast_sender, from_b_broadcast_receiver) = channel::unbounded(); + + // Node B is connected to the network with throuput of 1. + let to_b_receiver = + network.connect(node_b, Some(1), from_b_receiver, from_b_broadcast_receiver); + + // Every message sent **from** Node B will be of size 2. + let b = MockNetworkInterface::new( + node_b, + from_b_broadcast_sender, + from_b_sender, + to_b_receiver, + 15, + ); + + // Node A sends message of size 2 to Node B. + a.send_message(node_b, ()); + // Node B sends message of size 15 to Node A. + b.send_message(node_a, ()); + + // Step duration matches the latency between nodes, thus Node A can receive 5 units of a + // message, Node B - 1 unit of a message during the step. + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 0); + assert_eq!(b.receive_messages().len(), 0); + + // Node B should receive a message during the second step, because it's throughput during the + // step is 1, but the message size it receives is 2. + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 0); + assert_eq!(b.receive_messages().len(), 1); + + // Node A should receive a message during the third step, because it's throughput during the + // step is 5, but the message it recieves is of size 15. + network.step(Duration::from_millis(100)); + assert_eq!(a.receive_messages().len(), 1); + assert_eq!(b.receive_messages().len(), 0); + } +} diff --git a/network-runner/src/network/regions.rs b/network-runner/src/network/regions.rs new file mode 100644 index 0000000..328deb4 --- /dev/null +++ b/network-runner/src/network/regions.rs @@ -0,0 +1,235 @@ +// std +use rand::{seq::SliceRandom, Rng}; +use std::{collections::HashMap, str::FromStr}; +// crates +use serde::{Deserialize, Serialize}; +// internal +use crate::{network::behaviour::NetworkBehaviour, node::NodeId}; + +use super::{NetworkBehaviourKey, NetworkSettings}; + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +pub enum Region { + NorthAmerica, + Europe, + Asia, + Africa, + SouthAmerica, + Australia, +} + +impl core::fmt::Display for Region { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let s = match self { + Self::NorthAmerica => "NorthAmerica", + Self::Europe => "Europe", + Self::Asia => "Asia", + Self::Africa => "Africa", + Self::SouthAmerica => "SouthAmerica", + Self::Australia => "Australia", + }; + write!(f, "{s}") + } +} + +impl FromStr for Region { + type Err = String; + + fn from_str(s: &str) -> Result { + match s + .trim() + .to_lowercase() + .replace(['-', '_', ' '], "") + .as_str() + { + "northamerica" | "na" => Ok(Self::NorthAmerica), + "europe" | "eu" => Ok(Self::Europe), + "asia" | "as" => Ok(Self::Asia), + "africa" | "af" => Ok(Self::Africa), + "southamerica" | "sa" => Ok(Self::SouthAmerica), + "australia" | "au" => Ok(Self::Australia), + _ => Err(format!("Unknown region: {s}")), + } + } +} + +impl Serialize for Region { + fn serialize(&self, serializer: S) -> Result { + let s = match self { + Self::NorthAmerica => "North America", + Self::Europe => "Europe", + Self::Asia => "Asia", + Self::Africa => "Africa", + Self::SouthAmerica => "South America", + Self::Australia => "Australia", + }; + serializer.serialize_str(s) + } +} + +impl<'de> Deserialize<'de> for Region { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + Self::from_str(&s).map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RegionsData { + pub regions: HashMap>, + #[serde(skip)] + pub node_region: HashMap, + pub region_network_behaviour: HashMap, +} + +impl RegionsData { + pub fn new( + regions: HashMap>, + region_network_behaviour: HashMap, + ) -> Self { + let node_region = regions + .iter() + .flat_map(|(region, nodes)| nodes.iter().copied().map(|node| (node, *region))) + .collect(); + Self { + regions, + node_region, + region_network_behaviour, + } + } + + pub fn node_region(&self, node_id: NodeId) -> Region { + self.node_region[&node_id] + } + + pub fn network_behaviour(&self, node_a: NodeId, node_b: NodeId) -> &NetworkBehaviour { + let region_a = self.node_region[&node_a]; + let region_b = self.node_region[&node_b]; + let k = NetworkBehaviourKey::new(region_a, region_b); + let k_rev = NetworkBehaviourKey::new(region_b, region_a); + self.region_network_behaviour + .get(&k) + .or(self.region_network_behaviour.get(&k_rev)) + .expect("Network behaviour not found for the given regions") + } + + pub fn region_nodes(&self, region: Region) -> &[NodeId] { + &self.regions[®ion] + } +} + +// Takes a reference to the node_ids and simulation_settings and returns a HashMap +// representing the regions and their associated node IDs. +pub fn create_regions( + node_ids: &[NodeId], + rng: &mut R, + network_settings: &NetworkSettings, +) -> HashMap> { + let mut region_nodes = node_ids.to_vec(); + region_nodes.shuffle(rng); + + let regions = network_settings + .regions + .clone() + .into_iter() + .collect::>(); + + let last_region_index = regions.len() - 1; + + regions + .iter() + .enumerate() + .map(|(i, (region, distribution))| { + if i < last_region_index { + let node_count = (node_ids.len() as f32 * distribution).round() as usize; + let nodes = region_nodes.drain(..node_count).collect::>(); + (*region, nodes) + } else { + // Assign the remaining nodes to the last region. + (*region, region_nodes.clone()) + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use consensus_engine::NodeId; + use rand::rngs::mock::StepRng; + + use crate::{ + network::{ + regions::{create_regions, Region}, + NetworkSettings, + }, + node::NodeIdExt, + }; + + #[test] + fn create_regions_precision() { + struct TestCase { + node_count: usize, + distributions: Vec, + } + + let test_cases = vec![ + TestCase { + node_count: 10, + distributions: vec![0.5, 0.3, 0.2], + }, + TestCase { + node_count: 7, + distributions: vec![0.6, 0.4], + }, + TestCase { + node_count: 20, + distributions: vec![0.4, 0.3, 0.2, 0.1], + }, + TestCase { + node_count: 23, + distributions: vec![0.4, 0.3, 0.3], + }, + TestCase { + node_count: 111, + distributions: vec![0.3, 0.3, 0.3, 0.1], + }, + TestCase { + node_count: 73, + distributions: vec![0.3, 0.2, 0.2, 0.2, 0.1], + }, + ]; + let mut rng = StepRng::new(1, 0); + + for tcase in test_cases.iter() { + let nodes = (0..tcase.node_count) + .map(NodeId::from_index) + .collect::>(); + + let available_regions = [ + Region::NorthAmerica, + Region::Europe, + Region::Asia, + Region::Africa, + Region::SouthAmerica, + Region::Australia, + ]; + + let mut region_distribution = HashMap::new(); + for (region, &dist) in available_regions.iter().zip(&tcase.distributions) { + region_distribution.insert(*region, dist); + } + + let settings = NetworkSettings { + network_behaviors: HashMap::new(), + regions: region_distribution, + }; + + let regions = create_regions(&nodes, &mut rng, &settings); + + let total_nodes_in_regions = regions.values().map(|v| v.len()).sum::(); + assert_eq!(total_nodes_in_regions, nodes.len()); + } + } +} diff --git a/network-runner/src/node/dummy_streaming.rs b/network-runner/src/node/dummy_streaming.rs new file mode 100644 index 0000000..cdbf93b --- /dev/null +++ b/network-runner/src/node/dummy_streaming.rs @@ -0,0 +1,45 @@ +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +use super::{Node, NodeId}; + +#[derive(Debug, Default, Copy, Clone, Serialize, Deserialize)] +pub struct DummyStreamingState { + pub counter: usize, +} + +/// This node implementation only used for testing different streaming implementation purposes. +pub struct DummyStreamingNode { + id: NodeId, + state: DummyStreamingState, + #[allow(dead_code)] + settings: S, +} + +impl DummyStreamingNode { + pub fn new(id: NodeId, settings: S) -> Self { + Self { + id, + state: DummyStreamingState::default(), + settings, + } + } +} + +impl Node for DummyStreamingNode { + type Settings = S; + + type State = DummyStreamingState; + + fn id(&self) -> NodeId { + self.id + } + + fn state(&self) -> &Self::State { + &self.state + } + + fn step(&mut self, _: Duration) { + todo!() + } +} diff --git a/network-runner/src/node/mod.rs b/network-runner/src/node/mod.rs new file mode 100644 index 0000000..584c3ad --- /dev/null +++ b/network-runner/src/node/mod.rs @@ -0,0 +1,213 @@ +#[cfg(test)] +pub mod dummy_streaming; + +// std +use std::{ + collections::HashMap, + ops::{Deref, DerefMut}, + sync::Arc, + time::Duration, +}; +// crates +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; +// internal + +#[serde_with::serde_as] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct StepTime(#[serde_as(as = "serde_with::DurationMilliSeconds")] Duration); + +impl From for StepTime { + fn from(duration: Duration) -> Self { + Self(duration) + } +} + +impl StepTime { + #[inline] + pub const fn new(duration: Duration) -> Self { + Self(duration) + } + + #[inline] + pub const fn into_inner(&self) -> Duration { + self.0 + } + + #[inline] + pub const fn from_millis(millis: u64) -> Self { + Self(Duration::from_millis(millis)) + } + + #[inline] + pub const fn from_secs(secs: u64) -> Self { + Self(Duration::from_secs(secs)) + } +} + +impl Deref for StepTime { + type Target = Duration; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for StepTime { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl core::iter::Sum for StepTime { + fn sum>(iter: I) -> Self { + Self(iter.into_iter().map(|s| s.0).sum()) + } +} + +impl core::iter::Sum for StepTime { + fn sum>(iter: I) -> Self { + Self(iter.into_iter().sum()) + } +} + +impl core::iter::Sum for Duration { + fn sum>(iter: I) -> Self { + iter.into_iter().map(|s| s.0).sum() + } +} + +pub type SharedState = Arc>; + +pub type Step = usize; + +/// A state that represents how nodes are interconnected in the network. +pub struct OverlayState { + pub all_nodes: Vec, + pub overlay: MixnetOverlay, +} + +#[derive(Clone, Debug)] +pub struct MixnetOverlay { + pub connections: HashMap>, +} + +pub trait OverlayGetter { + fn get_overlay(&self) -> MixnetOverlay; + fn get_all_nodes(&self) -> Vec; +} + +impl OverlayGetter for SharedState { + fn get_overlay(&self) -> MixnetOverlay { + let overlay_state = self.read(); + overlay_state.clone() + } + + fn get_all_nodes(&self) -> Vec { + let overlay_state = self.read(); + overlay_state.connections.keys().cloned().collect() + } +} + +pub trait Node { + type Settings; + type State; + + fn id(&self) -> NodeId; + fn state(&self) -> &Self::State; + fn step(&mut self, elapsed: Duration); +} + +#[derive( + Clone, Copy, Default, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize, +)] +pub struct NodeId(pub [u8; 32]); + +impl NodeId { + pub const fn new(val: [u8; 32]) -> Self { + Self(val) + } + + /// Returns a random node id + pub fn random(rng: &mut R) -> Self { + let mut bytes = [0u8; 32]; + rng.fill_bytes(&mut bytes); + Self(bytes) + } +} + +impl From<[u8; 32]> for NodeId { + fn from(id: [u8; 32]) -> Self { + Self(id) + } +} + +impl From<&[u8; 32]> for NodeId { + fn from(id: &[u8; 32]) -> Self { + Self(*id) + } +} + +impl From for [u8; 32] { + fn from(id: NodeId) -> Self { + id.0 + } +} + +impl<'a> From<&'a NodeId> for &'a [u8; 32] { + fn from(id: &'a NodeId) -> Self { + &id.0 + } +} + +impl core::fmt::Display for NodeId { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "0x")?; + for v in self.0 { + write!(f, "{:02x}", v)?; + } + Ok(()) + } +} + +#[cfg(test)] +impl Node for usize { + type Settings = (); + type State = Self; + + fn id(&self) -> NodeId { + todo!() + } + + fn state(&self) -> &Self::State { + self + } + + fn step(&mut self, _: Duration) { + use std::ops::AddAssign; + self.add_assign(1); + } +} + +pub trait NodeIdExt { + fn index(&self) -> usize; + + fn from_index(idx: usize) -> Self; +} + +impl NodeIdExt for NodeId { + fn index(&self) -> usize { + const SIZE: usize = core::mem::size_of::(); + let mut bytes = [0u8; SIZE]; + let src: [u8; 32] = (*self).into(); + bytes.copy_from_slice(&src[..SIZE]); + usize::from_be_bytes(bytes) + } + + fn from_index(idx: usize) -> Self { + let mut bytes = [0u8; 32]; + bytes[..core::mem::size_of::()].copy_from_slice(&idx.to_be_bytes()); + NodeId::new(bytes) + } +} diff --git a/network-runner/src/output_processors/mod.rs b/network-runner/src/output_processors/mod.rs new file mode 100644 index 0000000..aa3cf1d --- /dev/null +++ b/network-runner/src/output_processors/mod.rs @@ -0,0 +1,132 @@ +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use serde::Serialize; + +use crate::settings::SimulationSettings; +use crate::warding::SimulationState; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum RecordType { + Meta, + Settings, + Data, +} + +pub trait Record: From + From + Send + Sync + 'static { + type Data: serde::Serialize; + + fn record_type(&self) -> RecordType; + + fn is_settings(&self) -> bool { + self.record_type() == RecordType::Settings + } + + fn is_meta(&self) -> bool { + self.record_type() == RecordType::Meta + } + + fn is_data(&self) -> bool { + self.record_type() == RecordType::Data + } + + fn data(&self) -> Vec<&Self::Data>; +} + +pub type SerializedNodeState = serde_json::Value; + +#[derive(Serialize)] +pub struct Runtime { + start: DateTime, + end: DateTime, + elapsed: Duration, +} + +impl Runtime { + pub(crate) fn load() -> anyhow::Result { + let elapsed = crate::START_TIME.elapsed(); + let end = Utc::now(); + Ok(Self { + start: end + .checked_sub_signed(chrono::Duration::from_std(elapsed)?) + .unwrap(), + end, + elapsed, + }) + } +} + +#[derive(Serialize)] +#[serde(untagged)] +pub enum OutData { + Runtime(Runtime), + Settings(Box), + Data(SerializedNodeState), +} + +impl From for OutData { + fn from(runtime: Runtime) -> Self { + Self::Runtime(runtime) + } +} + +impl From for OutData { + fn from(settings: SimulationSettings) -> Self { + Self::Settings(Box::new(settings)) + } +} + +impl From for OutData { + fn from(state: SerializedNodeState) -> Self { + Self::Data(state) + } +} + +impl Record for OutData { + type Data = SerializedNodeState; + + fn record_type(&self) -> RecordType { + match self { + Self::Runtime(_) => RecordType::Meta, + Self::Settings(_) => RecordType::Settings, + Self::Data(_) => RecordType::Data, + } + } + + fn data(&self) -> Vec<&SerializedNodeState> { + match self { + Self::Data(d) => vec![d], + _ => unreachable!(), + } + } +} + +impl OutData { + #[inline] + pub const fn new(state: SerializedNodeState) -> Self { + Self::Data(state) + } +} + +impl TryFrom<&SimulationState> for OutData { + type Error = anyhow::Error; + + fn try_from(state: &SimulationState) -> Result { + serde_json::to_value( + state + .nodes + .read() + .iter() + .map(|n| n.state()) + .collect::>(), + ) + .map(OutData::new) + .map_err(From::from) + } +} + +pub trait NodeStateRecord { + fn get_serialized_state_record(&self) -> SerializedNodeState { + SerializedNodeState::Null + } +} diff --git a/network-runner/src/runner/async_runner.rs b/network-runner/src/runner/async_runner.rs new file mode 100644 index 0000000..4cc5ddc --- /dev/null +++ b/network-runner/src/runner/async_runner.rs @@ -0,0 +1,75 @@ +use crate::node::NodeId; +use crate::output_processors::Record; +use crate::runner::SimulationRunner; +use crate::warding::SimulationState; +use crossbeam::channel::bounded; +use crossbeam::select; +use rand::prelude::SliceRandom; +use rayon::prelude::*; +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Duration; + +use super::SimulationRunnerHandle; + +/// Simulate with sending the network state to any subscriber +pub fn simulate( + runner: SimulationRunner, + chunk_size: usize, + step_time: Duration, +) -> anyhow::Result> +where + M: std::fmt::Debug + Clone + Send + Sync + 'static, + R: Record + + for<'a> TryFrom<&'a SimulationState, Error = anyhow::Error> + + Send + + Sync + + 'static, + S: 'static, + T: 'static, +{ + let simulation_state = SimulationState { + nodes: Arc::clone(&runner.nodes), + }; + + let mut node_ids: Vec = runner.nodes.read().iter().map(|n| n.id()).collect(); + + let mut inner_runner = runner.inner; + let nodes = runner.nodes; + let (stop_tx, stop_rx) = bounded(1); + let p = runner.producer.clone(); + let p1 = runner.producer; + let handle = std::thread::spawn(move || { + loop { + select! { + recv(stop_rx) -> _ => { + return Ok(()); + } + default => { + node_ids.shuffle(&mut inner_runner.rng); + for ids_chunk in node_ids.chunks(chunk_size) { + let ids: HashSet = ids_chunk.iter().copied().collect(); + nodes + .write() + .par_iter_mut() + .filter(|n| ids.contains(&n.id())) + .for_each(|node| node.step(step_time)); + + p.send(R::try_from( + &simulation_state, + )?)?; + } + // check if any condition makes the simulation stop + if inner_runner.check_wards(&simulation_state) { + return Ok(()); + } + } + } + } + }); + Ok(SimulationRunnerHandle { + producer: p1, + stop_tx, + handle, + }) +} diff --git a/network-runner/src/runner/mod.rs b/network-runner/src/runner/mod.rs new file mode 100644 index 0000000..cb002d5 --- /dev/null +++ b/network-runner/src/runner/mod.rs @@ -0,0 +1,214 @@ +mod async_runner; +mod sync_runner; + +// std +use std::sync::Arc; +use std::time::Duration; + +use crate::output_processors::Record; +// crates +use crate::streaming::{ + runtime_subscriber::RuntimeSubscriber, settings_subscriber::SettingsSubscriber, StreamProducer, + Subscriber, SubscriberHandle, +}; +use crossbeam::channel::Sender; +use parking_lot::RwLock; +use rand::rngs::SmallRng; +use rand::{RngCore, SeedableRng}; +use rayon::prelude::*; +use serde::Serialize; + +// internal +use crate::network::Network; +use crate::node::Node; +use crate::settings::{RunnerSettings, SimulationSettings}; +use crate::warding::{SimulationState, SimulationWard, Ward}; + +pub type BoxedNode = Box + Send + Sync>; + +pub struct SimulationRunnerHandle { + producer: StreamProducer, + stop_tx: Sender<()>, + handle: std::thread::JoinHandle>, +} + +impl SimulationRunnerHandle { + pub fn stop_after(self, duration: Duration) -> anyhow::Result<()> { + std::thread::sleep(duration); + self.stop() + } + + pub fn stop(&self) -> anyhow::Result<()> { + if !self.handle.is_finished() { + self.stop_tx.send(())?; + self.shutdown()?; + } + Ok(()) + } + + pub fn subscribe>( + &self, + settings: S::Settings, + ) -> anyhow::Result> { + self.producer.subscribe(settings) + } + + pub fn is_finished(&self) -> bool { + self.handle.is_finished() + } + + pub fn shutdown(&self) -> anyhow::Result<()> { + self.producer.stop() + } + + pub fn join(self) -> anyhow::Result<()> { + self.handle.join().expect("Join simulation thread") + } +} + +pub(crate) struct SimulationRunnerInner { + network: Network, + wards: Vec, + rng: SmallRng, +} + +impl SimulationRunnerInner +where + M: std::fmt::Debug + Send + Sync + Clone, +{ + fn check_wards(&mut self, state: &SimulationState) -> bool { + self.wards + .par_iter_mut() + .map(|ward| ward.analyze(state)) + .any(|x| x) + } + + fn step(&mut self, nodes: &mut [BoxedNode], elapsed: Duration) { + self.network.dispatch_after(elapsed); + nodes.par_iter_mut().for_each(|node| { + node.step(elapsed); + }); + self.network.collect_messages(); + } +} + +/// Encapsulation solution for the simulations runner +/// Holds the network state, the simulating nodes and the simulation settings. +pub struct SimulationRunner { + inner: SimulationRunnerInner, + nodes: Arc>>>, + runner_settings: RunnerSettings, + producer: StreamProducer, + step_time: Duration, +} + +impl SimulationRunner +where + M: std::fmt::Debug + Clone + Send + Sync + 'static, + R: Record + + for<'a> TryFrom<&'a SimulationState, Error = anyhow::Error> + + Send + + Sync + + 'static, + S: 'static, + T: Serialize + Clone + 'static, +{ + pub fn new( + network: Network, + nodes: Vec>, + producer: StreamProducer, + mut settings: SimulationSettings, + ) -> anyhow::Result { + let seed = settings + .seed + .unwrap_or_else(|| rand::thread_rng().next_u64()); + + settings + .seed + .get_or_insert_with(|| rand::thread_rng().next_u64()); + + // Store the settings to the producer so that we can collect them later + producer.send(R::from(settings.clone()))?; + + let rng = SmallRng::seed_from_u64(seed); + let nodes = Arc::new(RwLock::new(nodes)); + let SimulationSettings { + wards, + overlay_settings: _, + node_settings: _, + runner_settings, + stream_settings: _, + node_count: _, + seed: _, + views_count: _, + leaders_count: _, + network_settings: _, + step_time, + record_settings: _, + } = settings; + Ok(Self { + runner_settings, + inner: SimulationRunnerInner { + network, + rng, + wards, + }, + nodes, + producer, + step_time, + }) + } + + pub fn simulate(self) -> anyhow::Result> { + // init the start time + let _ = *crate::START_TIME; + let step_time = self.step_time; + + match self.runner_settings.clone() { + RunnerSettings::Sync => sync_runner::simulate(self, step_time), + } + } +} + +impl SimulationRunner +where + M: std::fmt::Debug + Clone + Send + Sync + 'static, + R: Record + + serde::Serialize + + for<'a> TryFrom<&'a SimulationState, Error = anyhow::Error> + + Send + + Sync + + 'static, + S: 'static, + T: Serialize + Clone + 'static, +{ + pub fn simulate_and_subscribe( + self, + settings: B::Settings, + ) -> anyhow::Result> + where + B: Subscriber + Send + Sync + 'static, + { + let handle = self.simulate()?; + let mut data_subscriber_handle = handle.subscribe::(settings)?; + let mut runtime_subscriber_handle = + handle.subscribe::>(Default::default())?; + let mut settings_subscriber_handle = + handle.subscribe::>(Default::default())?; + std::thread::scope(|s| { + s.spawn(move || { + data_subscriber_handle.run(); + }); + + s.spawn(move || { + runtime_subscriber_handle.run(); + }); + + s.spawn(move || { + settings_subscriber_handle.run(); + }); + }); + + Ok(handle) + } +} diff --git a/network-runner/src/runner/sync_runner.rs b/network-runner/src/runner/sync_runner.rs new file mode 100644 index 0000000..030f44e --- /dev/null +++ b/network-runner/src/runner/sync_runner.rs @@ -0,0 +1,236 @@ +use super::{SimulationRunner, SimulationRunnerHandle}; +use crate::output_processors::Record; +use crate::warding::SimulationState; +use crossbeam::channel::{bounded, select}; +use std::sync::Arc; +use std::time::Duration; + +/// Simulate with sending the network state to any subscriber +pub fn simulate( + runner: SimulationRunner, + step_time: Duration, +) -> anyhow::Result> +where + M: std::fmt::Debug + Send + Sync + Clone + 'static, + R: Record + + for<'a> TryFrom<&'a SimulationState, Error = anyhow::Error> + + Send + + Sync + + 'static, + S: 'static, + T: 'static, +{ + let state = SimulationState { + nodes: Arc::clone(&runner.nodes), + }; + + let mut inner_runner = runner.inner; + let nodes = runner.nodes; + + let (stop_tx, stop_rx) = bounded(1); + let p = runner.producer.clone(); + let p1 = runner.producer; + let handle = std::thread::spawn(move || { + p.send(R::try_from(&state)?)?; + loop { + select! { + recv(stop_rx) -> _ => { + return Ok(()); + } + default => { + // we must use a code block to make sure once the step call is finished then the write lock will be released, because in Record::try_from(&state), + // we need to call the read lock, if we do not release the write lock, + // then dead lock will occur + { + let mut nodes = nodes.write(); + inner_runner.step(&mut nodes, step_time); + } + + p.send(R::try_from(&state)?)?; + // check if any condition makes the simulation stop + if inner_runner.check_wards(&state) { + return Ok(()); + } + } + } + } + }); + Ok(SimulationRunnerHandle { + producer: p1, + stop_tx, + handle, + }) +} + +// #[cfg(test)] +// mod tests { +// use crate::{ +// network::{ +// behaviour::NetworkBehaviour, +// regions::{Region, RegionsData}, +// InMemoryNetworkInterface, Network, NetworkBehaviourKey, +// }, +// node::{Node, NodeId, NodeIdExt, OverlayState, SharedState}, +// output_processors::OutData, +// runner::SimulationRunner, +// settings::SimulationSettings, +// streaming::StreamProducer, +// }; +// use crossbeam::channel; +// use parking_lot::RwLock; +// use rand::rngs::mock::StepRng; +// use std::{ +// collections::{BTreeMap, HashMap}, +// sync::Arc, +// time::Duration, +// }; +// +// fn init_network(node_ids: &[NodeId]) -> Network { +// let regions = HashMap::from([(Region::Europe, node_ids.to_vec())]); +// let behaviour = HashMap::from([( +// NetworkBehaviourKey::new(Region::Europe, Region::Europe), +// NetworkBehaviour::new(Duration::from_millis(100), 0.0), +// )]); +// let regions_data = RegionsData::new(regions, behaviour); +// Network::new(regions_data, 0) +// } +// +// fn init_dummy_nodes( +// node_ids: &[NodeId], +// network: &mut Network, +// overlay_state: SharedState, +// ) -> Vec { +// node_ids +// .iter() +// .map(|node_id| { +// let (node_message_sender, node_message_receiver) = channel::unbounded(); +// let (node_message_broadcast_sender, node_message_broadcast_receiver) = +// channel::unbounded(); +// let network_message_receiver = network.connect( +// *node_id, +// Some(1), +// node_message_receiver, +// node_message_broadcast_receiver, +// ); +// let network_interface = InMemoryNetworkInterface::new( +// *node_id, +// node_message_broadcast_sender, +// node_message_sender, +// network_message_receiver, +// ); +// DummyNode::new( +// *node_id, +// View::new(0), +// overlay_state.clone(), +// network_interface, +// ) +// }) +// .collect() +// } +// +// #[test] +// fn runner_one_step() { +// let settings = SimulationSettings { +// node_count: 10, +// ..Default::default() +// }; +// +// let mut rng = StepRng::new(1, 0); +// let node_ids: Vec = (0..settings.node_count).map(NodeId::from_index).collect(); +// let overlay = TreeOverlay::new(TreeSettings::default()); +// let mut network = init_network(&node_ids); +// let view = ViewOverlay { +// leaders: overlay.leaders(&node_ids, 1, &mut rng).collect(), +// layout: overlay.layout(&node_ids, &mut rng), +// }; +// let overlay_state = Arc::new(RwLock::new(OverlayState { +// all_nodes: node_ids.clone(), +// overlay: SimulationOverlay::Tree(overlay), +// overlays: BTreeMap::from([(View::new(0), view.clone()), (View::new(1), view)]), +// })); +// let nodes = init_dummy_nodes(&node_ids, &mut network, overlay_state) +// .into_iter() +// .map(|n| { +// Box::new(n) +// as Box< +// dyn Node +// + std::marker::Send +// + Sync, +// > +// }) +// .collect(); +// +// let producer = StreamProducer::default(); +// let mut runner: SimulationRunner = +// SimulationRunner::<_, OutData, DummySettings, DummyState>::new( +// network, nodes, producer, settings, +// ) +// .unwrap(); +// let mut nodes = runner.nodes.write(); +// runner.inner.step(&mut nodes, Duration::from_millis(100)); +// drop(nodes); +// +// let nodes = runner.nodes.read(); +// for node in nodes.iter() { +// assert_eq!(node.current_view(), View::new(0)); +// } +// } +// +// #[test] +// fn runner_send_receive() { +// let settings = SimulationSettings { +// node_count: 10, +// ..Default::default() +// }; +// +// let mut rng = StepRng::new(1, 0); +// let node_ids: Vec = (0..settings.node_count).map(NodeId::from_index).collect(); +// let overlay = TreeOverlay::new(TreeSettings::default()); +// let mut network = init_network(&node_ids); +// let view = ViewOverlay { +// leaders: overlay.leaders(&node_ids, 1, &mut rng).collect(), +// layout: overlay.layout(&node_ids, &mut rng), +// }; +// let overlay_state = Arc::new(RwLock::new(OverlayState { +// all_nodes: node_ids.clone(), +// overlay: SimulationOverlay::Tree(overlay), +// overlays: BTreeMap::from([ +// (View::new(0), view.clone()), +// (View::new(1), view.clone()), +// (View::new(42), view.clone()), +// (View::new(43), view), +// ]), +// })); +// let nodes = init_dummy_nodes(&node_ids, &mut network, overlay_state.clone()); +// +// for node in nodes.iter() { +// // All nodes send one message to NodeId(1). +// // Nodes can send messages to themselves. +// node.send_message(node_ids[1], DummyMessage::Proposal(View::new(42).into())); +// } +// network.collect_messages(); +// +// let nodes = init_dummy_nodes(&node_ids, &mut network, overlay_state) +// .into_iter() +// .map(|n| { +// Box::new(n) +// as Box< +// dyn Node +// + std::marker::Send +// + Sync, +// > +// }) +// .collect(); +// +// let mut runner: SimulationRunner = +// SimulationRunner::new(network, nodes, Default::default(), settings).unwrap(); +// +// let mut nodes = runner.nodes.write(); +// runner.inner.step(&mut nodes, Duration::from_millis(100)); +// drop(nodes); +// +// let nodes = runner.nodes.read(); +// let state = nodes[1].state(); +// assert_eq!(state.message_count, 10); +// } +// } diff --git a/network-runner/src/settings.rs b/network-runner/src/settings.rs new file mode 100644 index 0000000..cdeb889 --- /dev/null +++ b/network-runner/src/settings.rs @@ -0,0 +1,58 @@ +use std::collections::BTreeMap; + +use crate::network::NetworkSettings; +use crate::streaming::StreamSettings; +use crate::warding::Ward; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub enum RunnerSettings { + #[default] + Sync, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(untagged)] +pub enum OverlaySettings { + #[default] + Flat, + Tree(TreeSettings), + Branch(BranchSettings), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TreeSettings { + pub number_of_committees: usize, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct BranchSettings { + pub branch_depth: usize, +} + +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct NodeSettings { + pub network_capacity_kbps: Option, + #[serde(with = "humantime_serde")] + pub timeout: std::time::Duration, +} + +#[derive(Clone, Default, Debug, Serialize, Deserialize)] +pub struct SimulationSettings { + #[serde(default)] + pub wards: Vec, + #[serde(default)] + pub record_settings: BTreeMap, + pub network_settings: NetworkSettings, + pub overlay_settings: OverlaySettings, + pub node_settings: NodeSettings, + #[serde(default)] + pub runner_settings: RunnerSettings, + pub stream_settings: StreamSettings, + #[serde(with = "humantime_serde")] + pub step_time: std::time::Duration, + pub node_count: usize, + pub views_count: usize, + pub leaders_count: usize, + pub seed: Option, +} diff --git a/network-runner/src/streaming/io.rs b/network-runner/src/streaming/io.rs new file mode 100644 index 0000000..9332731 --- /dev/null +++ b/network-runner/src/streaming/io.rs @@ -0,0 +1,235 @@ +use std::{any::Any, io::stdout, sync::Arc}; + +use super::{Receivers, StreamSettings, Subscriber}; +use crate::output_processors::{RecordType, Runtime}; +use crossbeam::channel::{Receiver, Sender}; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct IOStreamSettings { + #[serde(rename = "type")] + pub writer_type: WriteType, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WriteType { + #[default] + Stdout, +} + +pub trait ToWriter { + fn to_writer(&self) -> anyhow::Result; +} + +impl ToWriter for WriteType { + fn to_writer(&self) -> anyhow::Result { + match self { + WriteType::Stdout => { + let boxed_any = Box::new(stdout()) as Box; + Ok(boxed_any.downcast::().map(|boxed| *boxed).map_err(|_| { + std::io::Error::new(std::io::ErrorKind::Other, "Writer type mismatch") + })?) + } + } + } +} + +impl TryFrom for IOStreamSettings { + type Error = String; + + fn try_from(settings: StreamSettings) -> Result { + match settings { + StreamSettings::IO(settings) => Ok(settings), + _ => Err("io settings can't be created".into()), + } + } +} + +#[derive(Debug)] +pub struct IOSubscriber { + recvs: Arc>, + writer: Arc>, +} + +impl Subscriber for IOSubscriber +where + W: std::io::Write + Send + Sync + 'static, + R: crate::output_processors::Record + Serialize, +{ + type Record = R; + type Settings = IOStreamSettings; + + fn new( + record_recv: Receiver>, + stop_recv: Receiver>, + settings: Self::Settings, + ) -> anyhow::Result + where + Self: Sized, + { + Ok(Self { + recvs: Arc::new(Receivers { + stop_rx: stop_recv, + recv: record_recv, + }), + writer: Arc::new(Mutex::new(settings.writer_type.to_writer()?)), + }) + } + + fn next(&self) -> Option>> { + Some(self.recvs.recv.recv().map_err(From::from)) + } + + fn run(self) -> anyhow::Result<()> { + loop { + crossbeam::select! { + recv(self.recvs.stop_rx) -> finish_tx => { + // Flush remaining messages after stop signal. + while let Ok(msg) = self.recvs.recv.try_recv() { + self.sink(msg)?; + } + + // collect the run time meta + self.sink(Arc::new(R::from(Runtime::load()?)))?; + + finish_tx?.send(())? + } + recv(self.recvs.recv) -> msg => { + self.sink(msg?)?; + } + } + } + } + + fn sink(&self, state: Arc) -> anyhow::Result<()> { + serde_json::to_writer(&mut *self.writer.lock(), &state)?; + Ok(()) + } + + fn subscribe_data_type() -> RecordType { + RecordType::Data + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, time::Duration}; + + use consensus_engine::View; + + use crate::{ + network::{ + behaviour::NetworkBehaviour, + regions::{Region, RegionsData}, + Network, NetworkBehaviourKey, + }, + node::{ + dummy_streaming::{DummyStreamingNode, DummyStreamingState}, + Node, NodeId, NodeIdExt, + }, + output_processors::OutData, + runner::SimulationRunner, + warding::SimulationState, + }; + + use super::*; + #[derive(Debug, Clone, Serialize)] + struct IORecord { + states: HashMap, + } + + impl TryFrom<&SimulationState> for IORecord { + type Error = anyhow::Error; + + fn try_from(value: &SimulationState) -> Result { + let nodes = value.nodes.read(); + Ok(Self { + states: nodes + .iter() + .map(|node| (node.id(), node.current_view())) + .collect(), + }) + } + } + + #[test] + fn test_streaming() { + let simulation_settings = crate::settings::SimulationSettings { + seed: Some(1), + ..Default::default() + }; + + let nodes = (0..6) + .map(|idx| { + Box::new(DummyStreamingNode::new(NodeId::from_index(idx), ())) + as Box< + dyn Node + + std::marker::Send + + Sync, + > + }) + .collect::>(); + let network = Network::new( + RegionsData { + regions: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + (region, vec![NodeId::from_index(idx)]) + }) + .collect(), + node_region: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + (NodeId::from_index(idx), region) + }) + .collect(), + region_network_behaviour: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + ( + NetworkBehaviourKey::new(region, region), + NetworkBehaviour { + delay: Duration::from_millis(100), + drop: 0.0, + }, + ) + }) + .collect(), + }, + 0, + ); + let simulation_runner: SimulationRunner<(), OutData, (), DummyStreamingState> = + SimulationRunner::new(network, nodes, Default::default(), simulation_settings).unwrap(); + simulation_runner + .simulate() + .unwrap() + .stop_after(Duration::from_millis(100)) + .unwrap(); + } +} diff --git a/network-runner/src/streaming/mod.rs b/network-runner/src/streaming/mod.rs new file mode 100644 index 0000000..946cf94 --- /dev/null +++ b/network-runner/src/streaming/mod.rs @@ -0,0 +1,367 @@ +use std::{ + str::FromStr, + sync::{Arc, Mutex}, + time::Duration, +}; + +use crossbeam::channel::{bounded, unbounded, Receiver, Sender}; +use serde::{Deserialize, Serialize}; + +use crate::output_processors::{Record, RecordType, Runtime}; + +pub mod io; +pub mod naive; +#[cfg(feature = "polars")] +pub mod polars; +pub mod runtime_subscriber; +pub mod settings_subscriber; + +#[derive(Debug, Default, Clone, Copy, Serialize, PartialEq, Eq)] +pub enum SubscriberFormat { + Json, + #[default] + Csv, + Parquet, +} + +impl SubscriberFormat { + pub const fn csv() -> Self { + Self::Csv + } + + pub const fn json() -> Self { + Self::Json + } + + pub const fn parquet() -> Self { + Self::Parquet + } + + pub fn is_csv(&self) -> bool { + matches!(self, Self::Csv) + } +} + +impl FromStr for SubscriberFormat { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "json" => Ok(Self::Json), + "csv" => Ok(Self::Csv), + "parquet" => Ok(Self::Parquet), + tag => Err(format!( + "Invalid {tag} format, only [json, csv, parquet] are supported", + )), + } + } +} + +impl<'de> Deserialize<'de> for SubscriberFormat { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + SubscriberFormat::from_str(&s).map_err(serde::de::Error::custom) + } +} + +pub enum SubscriberType { + Meta, + Settings, + Data, +} + +#[derive(Debug)] +struct Receivers { + stop_rx: Receiver>, + recv: Receiver>, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize)] +pub enum StreamType { + #[default] + IO, + Naive, + #[cfg(feature = "polars")] + Polars, +} + +impl FromStr for StreamType { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "io" => Ok(Self::IO), + "naive" => Ok(Self::Naive), + #[cfg(feature = "polars")] + "polars" => Ok(Self::Polars), + tag => Err(format!( + "Invalid {tag} streaming type, only [naive, polars] are supported", + )), + } + } +} + +impl<'de> serde::Deserialize<'de> for StreamType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + StreamType::from_str(&s).map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase", untagged)] +pub enum StreamSettings { + Naive(naive::NaiveSettings), + IO(io::IOStreamSettings), + #[cfg(feature = "polars")] + Polars(polars::PolarsSettings), +} + +impl Default for StreamSettings { + fn default() -> Self { + Self::IO(Default::default()) + } +} + +impl StreamSettings { + pub fn unwrap_naive(self) -> naive::NaiveSettings { + match self { + StreamSettings::Naive(settings) => settings, + _ => panic!("unwrap naive failed"), + } + } + + pub fn unwrap_io(self) -> io::IOStreamSettings { + match self { + StreamSettings::IO(settings) => settings, + _ => panic!("unwrap io failed"), + } + } + + #[cfg(feature = "polars")] + pub fn unwrap_polars(self) -> polars::PolarsSettings { + match self { + StreamSettings::Polars(settings) => settings, + _ => panic!("unwrap polars failed"), + } + } +} + +pub struct SubscriberHandle { + handle: Option>>, + stop_tx: Sender>, + subscriber: Option, +} + +impl SubscriberHandle +where + S: Subscriber + Send + 'static, +{ + pub fn run(&mut self) { + if self.handle.is_some() { + return; + } + + // unwrap safe here, because if handld is none, then we must have not booted the subscriber. + let subscriber = self.subscriber.take().unwrap(); + let handle = std::thread::spawn(move || subscriber.run()); + self.handle = Some(handle); + } + + pub fn stop_after(self, duration: Duration) -> anyhow::Result<()> { + std::thread::sleep(duration); + self.stop() + } + + pub fn stop(self) -> anyhow::Result<()> { + if let Some(handle) = self.handle { + // if we have a handle, and the handle is not finished + if !handle.is_finished() { + let (finish_tx, finish_rx) = bounded(1); + self.stop_tx.send(finish_tx)?; + finish_rx.recv()?; + } else { + // we are sure the handle is finished, so we can join it and try to get the result. + // if we have any error on subscriber side, return the error. + match handle.join() { + Ok(rst) => rst?, + Err(_) => { + tracing::error!("Error joining subscriber thread"); + } + } + } + Ok(()) + } else { + // if we do not have a handle, then we have not booted the subscriber yet. + // we can just return immediately + Ok(()) + } + } +} + +#[derive(Debug)] +struct Senders { + record_ty: RecordType, + record_sender: Sender>, + stop_sender: Sender>, +} + +#[derive(Debug)] +struct StreamProducerInner { + /// senders is used to send messages to subscribers. + senders: Vec>, + + /// record_cache is used to cache messsages when there are no subscribers. + record_cache: Vec>, +} + +impl Default for StreamProducerInner { + fn default() -> Self { + Self { + senders: Vec::new(), + record_cache: Vec::new(), + } + } +} + +#[derive(Debug)] +pub struct StreamProducer { + inner: Arc>>, +} + +impl Default for StreamProducer { + fn default() -> Self { + Self { + inner: Arc::new(Mutex::new(StreamProducerInner::default())), + } + } +} + +impl Clone for StreamProducer { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + +impl StreamProducer { + pub fn new() -> Self { + Self::default() + } +} + +impl StreamProducer +where + R: Record + Send + Sync + 'static, +{ + pub fn send(&self, record: R) -> anyhow::Result<()> { + let mut inner = self.inner.lock().unwrap(); + if inner.senders.is_empty() { + inner.record_cache.push(Arc::new(record)); + Ok(()) + } else { + let record = Arc::new(record); + // cache record for new subscriber + inner.record_cache.push(record.clone()); + + // if a send fails, then it means the corresponding subscriber is dropped, + // we just remove the sender from the list of senders. + inner.senders.retain(|tx| { + if tx.record_ty != record.record_type() { + true + } else { + tx.record_sender.send(Arc::clone(&record)).is_ok() + } + }); + Ok(()) + } + } + + pub fn subscribe>( + &self, + settings: S::Settings, + ) -> anyhow::Result> { + let (tx, rx) = unbounded(); + let (stop_tx, stop_rx) = bounded(1); + let mut inner = self.inner.lock().unwrap(); + + // send all previous records to the new subscriber + for record in inner.record_cache.iter() { + if S::subscribe_data_type() == record.record_type() { + tx.send(Arc::clone(record))?; + } + } + + inner.senders.push(Senders { + record_sender: tx, + stop_sender: stop_tx.clone(), + record_ty: S::subscribe_data_type(), + }); + Ok(SubscriberHandle { + handle: None, + stop_tx, + subscriber: Some(S::new(rx, stop_rx, settings)?), + }) + } + + pub fn stop(&self) -> anyhow::Result<()> { + let meta_record = Arc::new(R::from(Runtime::load()?)); + let inner = self.inner.lock().unwrap(); + + // send runtime record to runtime subscribers + inner.senders.iter().for_each(|tx| { + if tx.record_ty == meta_record.record_type() { + if let Err(e) = tx.record_sender.send(Arc::clone(&meta_record)) { + tracing::error!("Error sending meta record: {e}"); + } + } + }); + + // send stop signal to all subscribers + inner.senders.iter().for_each(|tx| { + let (finish_tx, finish_rx) = bounded(1); + if let Err(e) = tx.stop_sender.send(finish_tx) { + tracing::error!("Error stopping subscriber: {e}"); + } else if let Err(e) = finish_rx.recv() { + tracing::error!("Error finilizing subscriber: {e}"); + } + }); + Ok(()) + } +} + +pub trait Subscriber { + type Settings; + type Record: crate::output_processors::Record + Serialize; + + fn new( + record_recv: Receiver>, + stop_recv: Receiver>, + settings: Self::Settings, + ) -> anyhow::Result + where + Self: Sized; + + fn next(&self) -> Option>>; + + fn run(self) -> anyhow::Result<()> + where + Self: Sized, + { + while let Some(state) = self.next() { + self.sink(state?)?; + } + Ok(()) + } + + fn sink(&self, state: Arc) -> anyhow::Result<()>; + + fn subscribe_data_type() -> RecordType; +} diff --git a/network-runner/src/streaming/naive.rs b/network-runner/src/streaming/naive.rs new file mode 100644 index 0000000..6b118ce --- /dev/null +++ b/network-runner/src/streaming/naive.rs @@ -0,0 +1,311 @@ +use super::{Receivers, StreamSettings, Subscriber, SubscriberFormat}; +use crate::output_processors::{Record, RecordType, Runtime}; +use crossbeam::channel::{Receiver, Sender}; +use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{File, OpenOptions}, + io::{Seek, Write}, + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NaiveSettings { + pub path: PathBuf, + #[serde(default = "SubscriberFormat::csv")] + pub format: SubscriberFormat, +} + +impl TryFrom for NaiveSettings { + type Error = String; + + fn try_from(settings: StreamSettings) -> Result { + match settings { + StreamSettings::Naive(settings) => Ok(settings), + _ => Err("naive settings can't be created".into()), + } + } +} + +impl Default for NaiveSettings { + fn default() -> Self { + let mut tmp = std::env::temp_dir(); + tmp.push("simulation"); + tmp.set_extension("data"); + Self { + path: tmp, + format: SubscriberFormat::Csv, + } + } +} + +#[derive(Debug)] +pub struct NaiveSubscriber { + file: Mutex, + recvs: Receivers, + initialized: AtomicBool, + format: SubscriberFormat, +} + +impl Subscriber for NaiveSubscriber +where + R: crate::output_processors::Record + Serialize, +{ + type Record = R; + + type Settings = NaiveSettings; + + fn new( + record_recv: Receiver>, + stop_recv: Receiver>, + settings: Self::Settings, + ) -> anyhow::Result + where + Self: Sized, + { + let mut opts = OpenOptions::new(); + let recvs = Receivers { + stop_rx: stop_recv, + recv: record_recv, + }; + let this = NaiveSubscriber { + file: Mutex::new( + opts.truncate(true) + .create(true) + .read(true) + .write(true) + .open(&settings.path)?, + ), + recvs, + initialized: AtomicBool::new(false), + format: settings.format, + }; + tracing::info!( + target = "simulation", + "subscribed to {}", + settings.path.display() + ); + Ok(this) + } + + fn next(&self) -> Option>> { + Some(self.recvs.recv.recv().map_err(From::from)) + } + + fn run(self) -> anyhow::Result<()> { + loop { + crossbeam::select! { + recv(self.recvs.stop_rx) -> finish_tx => { + // Flush remaining messages after stop signal. + while let Ok(msg) = self.recvs.recv.try_recv() { + self.sink(msg)?; + } + + // collect the run time meta + self.sink(Arc::new(R::from(Runtime::load()?)))?; + + finish_tx?.send(())? + } + recv(self.recvs.recv) -> msg => { + self.sink(msg?)?; + } + } + } + } + + fn sink(&self, state: Arc) -> anyhow::Result<()> { + let mut file = self.file.lock(); + match self.format { + SubscriberFormat::Json => { + write_json_record(&mut *file, &self.initialized, &*state)?; + } + SubscriberFormat::Csv => { + write_csv_record(&mut *file, &self.initialized, &*state)?; + } + SubscriberFormat::Parquet => { + panic!("native subscriber does not support parquet format") + } + } + + Ok(()) + } + + fn subscribe_data_type() -> RecordType { + RecordType::Data + } +} + +impl Drop for NaiveSubscriber { + fn drop(&mut self) { + if SubscriberFormat::Json == self.format { + let mut file = self.file.lock(); + // To construct a valid json format, we need to overwrite the last comma + if let Err(e) = file + .seek(std::io::SeekFrom::End(-1)) + .and_then(|_| file.write_all(b"]}")) + { + tracing::error!(target="simulations", err=%e, "fail to close json format"); + } + } + } +} + +fn write_json_record( + mut w: W, + initialized: &AtomicBool, + record: &R, +) -> std::io::Result<()> { + if !initialized.load(Ordering::Acquire) { + w.write_all(b"{\"records\": [")?; + initialized.store(true, Ordering::Release); + } + for data in record.data() { + serde_json::to_writer(&mut w, data)?; + w.write_all(b",")?; + } + Ok(()) +} + +fn write_csv_record( + w: &mut W, + initialized: &AtomicBool, + record: &R, +) -> csv::Result<()> { + // If have not write csv header, then write it + let mut w = if !initialized.load(Ordering::Acquire) { + initialized.store(true, Ordering::Release); + csv::WriterBuilder::new().has_headers(true).from_writer(w) + } else { + csv::WriterBuilder::new().has_headers(false).from_writer(w) + }; + for data in record.data() { + w.serialize(data).map_err(|e| { + tracing::error!(target = "simulations", err = %e, "fail to write CSV record"); + e + })?; + w.flush()?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, time::Duration}; + + use consensus_engine::View; + + use crate::{ + network::{ + behaviour::NetworkBehaviour, + regions::{Region, RegionsData}, + Network, NetworkBehaviourKey, + }, + node::{ + dummy_streaming::{DummyStreamingNode, DummyStreamingState}, + Node, NodeId, NodeIdExt, + }, + output_processors::OutData, + runner::SimulationRunner, + warding::SimulationState, + }; + + use super::*; + #[derive(Debug, Clone, Serialize)] + struct NaiveRecord { + states: HashMap, + } + + impl TryFrom<&SimulationState> for NaiveRecord { + type Error = anyhow::Error; + + fn try_from(value: &SimulationState) -> Result { + Ok(Self { + states: value + .nodes + .read() + .iter() + .map(|node| (node.id(), node.current_view())) + .collect(), + }) + } + } + + #[test] + fn test_streaming() { + let simulation_settings = crate::settings::SimulationSettings { + seed: Some(1), + ..Default::default() + }; + + let nodes = (0..6) + .map(|idx| { + Box::new(DummyStreamingNode::new(NodeId::from_index(idx), ())) + as Box< + dyn Node + + std::marker::Send + + Sync, + > + }) + .collect::>(); + let network = Network::new( + RegionsData { + regions: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + (region, vec![NodeId::from_index(idx)]) + }) + .collect(), + node_region: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + (NodeId::from_index(idx), region) + }) + .collect(), + region_network_behaviour: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + ( + NetworkBehaviourKey::new(region, region), + NetworkBehaviour { + delay: Duration::from_millis(100), + drop: 0.0, + }, + ) + }) + .collect(), + }, + 0, + ); + let simulation_runner: SimulationRunner<(), OutData, (), DummyStreamingState> = + SimulationRunner::new(network, nodes, Default::default(), simulation_settings).unwrap(); + simulation_runner.simulate().unwrap(); + } +} diff --git a/network-runner/src/streaming/polars.rs b/network-runner/src/streaming/polars.rs new file mode 100644 index 0000000..20e2c26 --- /dev/null +++ b/network-runner/src/streaming/polars.rs @@ -0,0 +1,156 @@ +use super::{Receivers, StreamSettings, SubscriberFormat}; +use crate::output_processors::{RecordType, Runtime}; +use crossbeam::channel::{Receiver, Sender}; +use parking_lot::Mutex; +use polars::prelude::*; +use serde::{Deserialize, Serialize}; +use std::{ + fs::File, + io::Cursor, + path::{Path, PathBuf}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PolarsSettings { + pub format: SubscriberFormat, + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +impl TryFrom for PolarsSettings { + type Error = String; + + fn try_from(settings: StreamSettings) -> Result { + match settings { + StreamSettings::Polars(settings) => Ok(settings), + _ => Err("polars settings can't be created".into()), + } + } +} + +#[derive(Debug)] +pub struct PolarsSubscriber { + data: Mutex>>, + path: PathBuf, + format: SubscriberFormat, + recvs: Receivers, +} + +impl PolarsSubscriber +where + R: Serialize, +{ + fn persist(&self) -> anyhow::Result<()> { + let data = self.data.lock(); + let mut cursor = Cursor::new(Vec::new()); + serde_json::to_writer(&mut cursor, &*data).expect("Dump data to json "); + let mut data = JsonReader::new(cursor) + .finish() + .expect("Load dataframe from intermediary json"); + + data.unnest(["state"])?; + match self.format { + SubscriberFormat::Json => dump_dataframe_to_json(&mut data, self.path.as_path()), + SubscriberFormat::Csv => dump_dataframe_to_csv(&mut data, self.path.as_path()), + SubscriberFormat::Parquet => dump_dataframe_to_parquet(&mut data, self.path.as_path()), + } + } +} + +impl super::Subscriber for PolarsSubscriber +where + R: crate::output_processors::Record + Serialize, +{ + type Record = R; + type Settings = PolarsSettings; + + fn new( + record_recv: Receiver>, + stop_recv: Receiver>, + settings: Self::Settings, + ) -> anyhow::Result + where + Self: Sized, + { + let recvs = Receivers { + stop_rx: stop_recv, + recv: record_recv, + }; + let this = PolarsSubscriber { + data: Mutex::new(Vec::new()), + recvs, + path: settings.path.clone().unwrap_or_else(|| { + let mut p = std::env::temp_dir().join("polars"); + match settings.format { + SubscriberFormat::Json => p.set_extension("json"), + SubscriberFormat::Csv => p.set_extension("csv"), + SubscriberFormat::Parquet => p.set_extension("parquet"), + }; + p + }), + format: settings.format, + }; + tracing::info!( + target = "simulation", + "subscribed to {}", + this.path.display() + ); + Ok(this) + } + + fn next(&self) -> Option>> { + Some(self.recvs.recv.recv().map_err(From::from)) + } + + fn run(self) -> anyhow::Result<()> { + loop { + crossbeam::select! { + recv(self.recvs.stop_rx) -> finish_tx => { + // Flush remaining messages after stop signal. + while let Ok(msg) = self.recvs.recv.try_recv() { + self.sink(msg)?; + } + + // collect the run time meta + self.sink(Arc::new(R::from(Runtime::load()?)))?; + + finish_tx?.send(())?; + return self.persist(); + } + recv(self.recvs.recv) -> msg => { + self.sink(msg?)?; + } + } + } + } + + fn sink(&self, state: Arc) -> anyhow::Result<()> { + self.data.lock().push(state); + Ok(()) + } + + fn subscribe_data_type() -> RecordType { + RecordType::Data + } +} + +fn dump_dataframe_to_json(data: &mut DataFrame, out_path: &Path) -> anyhow::Result<()> { + let out_path = out_path.with_extension("json"); + let f = File::create(out_path)?; + let mut writer = polars::prelude::JsonWriter::new(f); + Ok(writer.finish(data)?) +} + +fn dump_dataframe_to_csv(data: &mut DataFrame, out_path: &Path) -> anyhow::Result<()> { + let out_path = out_path.with_extension("csv"); + let f = File::create(out_path)?; + let mut writer = polars::prelude::CsvWriter::new(f); + Ok(writer.finish(data)?) +} + +fn dump_dataframe_to_parquet(data: &mut DataFrame, out_path: &Path) -> anyhow::Result<()> { + let out_path = out_path.with_extension("parquet"); + let f = File::create(out_path)?; + let writer = polars::prelude::ParquetWriter::new(f); + Ok(writer.finish(data).map(|_| ())?) +} diff --git a/network-runner/src/streaming/runtime_subscriber.rs b/network-runner/src/streaming/runtime_subscriber.rs new file mode 100644 index 0000000..f4c36be --- /dev/null +++ b/network-runner/src/streaming/runtime_subscriber.rs @@ -0,0 +1,218 @@ +use super::{Receivers, Subscriber}; +use crate::output_processors::{RecordType, Runtime}; +use crossbeam::channel::{Receiver, Sender}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{File, OpenOptions}, + io::Write, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RuntimeSettings { + pub path: PathBuf, +} + +impl Default for RuntimeSettings { + fn default() -> Self { + let mut tmp = std::env::temp_dir(); + tmp.push("simulation"); + tmp.set_extension("runtime"); + Self { path: tmp } + } +} + +#[derive(Debug)] +pub struct RuntimeSubscriber { + file: Arc>, + recvs: Arc>, +} + +impl Subscriber for RuntimeSubscriber +where + R: crate::output_processors::Record + Serialize, +{ + type Record = R; + + type Settings = RuntimeSettings; + + fn new( + record_recv: Receiver>, + stop_recv: Receiver>, + settings: Self::Settings, + ) -> anyhow::Result + where + Self: Sized, + { + let mut opts = OpenOptions::new(); + let recvs = Receivers { + stop_rx: stop_recv, + recv: record_recv, + }; + let this = RuntimeSubscriber { + file: Arc::new(Mutex::new( + opts.truncate(true) + .create(true) + .read(true) + .write(true) + .open(&settings.path)?, + )), + recvs: Arc::new(recvs), + }; + tracing::info!( + taget = "simulation", + "subscribed to {}", + settings.path.display() + ); + Ok(this) + } + + fn next(&self) -> Option>> { + Some(self.recvs.recv.recv().map_err(From::from)) + } + + fn run(self) -> anyhow::Result<()> { + crossbeam::select! { + recv(self.recvs.stop_rx) -> finish_tx => { + // collect the run time meta + self.sink(Arc::new(R::from(Runtime::load()?)))?; + finish_tx?.send(())?; + } + recv(self.recvs.recv) -> msg => { + self.sink(msg?)?; + } + } + + Ok(()) + } + + fn sink(&self, state: Arc) -> anyhow::Result<()> { + let mut file = self.file.lock().expect("failed to lock file"); + serde_json::to_writer(&mut *file, &state)?; + file.write_all(b",\n")?; + Ok(()) + } + + fn subscribe_data_type() -> RecordType { + RecordType::Data + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, time::Duration}; + + use consensus_engine::View; + + use crate::{ + network::{ + behaviour::NetworkBehaviour, + regions::{Region, RegionsData}, + Network, NetworkBehaviourKey, + }, + node::{ + dummy_streaming::{DummyStreamingNode, DummyStreamingState}, + Node, NodeId, NodeIdExt, + }, + output_processors::OutData, + runner::SimulationRunner, + warding::SimulationState, + }; + + use super::*; + #[derive(Debug, Clone, Serialize)] + struct RuntimeRecord { + states: HashMap, + } + + impl TryFrom<&SimulationState> for RuntimeRecord { + type Error = anyhow::Error; + + fn try_from(value: &SimulationState) -> Result { + Ok(Self { + states: value + .nodes + .read() + .iter() + .map(|node| (node.id(), node.current_view())) + .collect(), + }) + } + } + + #[test] + fn test_streaming() { + let simulation_settings = crate::settings::SimulationSettings { + seed: Some(1), + ..Default::default() + }; + + let nodes = (0..6) + .map(|idx| { + Box::new(DummyStreamingNode::new(NodeId::from_index(idx), ())) + as Box< + dyn Node + + std::marker::Send + + Sync, + > + }) + .collect::>(); + let network = Network::new( + RegionsData { + regions: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + (region, vec![NodeId::from_index(idx)]) + }) + .collect(), + node_region: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + (NodeId::from_index(idx), region) + }) + .collect(), + region_network_behaviour: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + ( + NetworkBehaviourKey::new(region, region), + NetworkBehaviour { + delay: Duration::from_millis(100), + drop: 0.0, + }, + ) + }) + .collect(), + }, + 0, + ); + let simulation_runner: SimulationRunner<(), OutData, (), DummyStreamingState> = + SimulationRunner::new(network, nodes, Default::default(), simulation_settings).unwrap(); + simulation_runner.simulate().unwrap(); + } +} diff --git a/network-runner/src/streaming/settings_subscriber.rs b/network-runner/src/streaming/settings_subscriber.rs new file mode 100644 index 0000000..2d3aa07 --- /dev/null +++ b/network-runner/src/streaming/settings_subscriber.rs @@ -0,0 +1,218 @@ +use super::{Receivers, Subscriber}; +use crate::output_processors::{RecordType, Runtime}; +use crossbeam::channel::{Receiver, Sender}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{File, OpenOptions}, + io::Write, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SettingsSubscriberSettings { + pub path: PathBuf, +} + +impl Default for SettingsSubscriberSettings { + fn default() -> Self { + let mut tmp = std::env::temp_dir(); + tmp.push("simulation"); + tmp.set_extension("conf"); + Self { path: tmp } + } +} + +#[derive(Debug)] +pub struct SettingsSubscriber { + file: Arc>, + recvs: Arc>, +} + +impl Subscriber for SettingsSubscriber +where + R: crate::output_processors::Record + Serialize, +{ + type Record = R; + + type Settings = SettingsSubscriberSettings; + + fn new( + record_recv: Receiver>, + stop_recv: Receiver>, + settings: Self::Settings, + ) -> anyhow::Result + where + Self: Sized, + { + let mut opts = OpenOptions::new(); + let recvs = Receivers { + stop_rx: stop_recv, + recv: record_recv, + }; + let this = SettingsSubscriber { + file: Arc::new(Mutex::new( + opts.truncate(true) + .create(true) + .read(true) + .write(true) + .open(&settings.path)?, + )), + recvs: Arc::new(recvs), + }; + tracing::info!( + target = "simulation", + "subscribed to {}", + settings.path.display() + ); + Ok(this) + } + + fn next(&self) -> Option>> { + Some(self.recvs.recv.recv().map_err(From::from)) + } + + fn run(self) -> anyhow::Result<()> { + crossbeam::select! { + recv(self.recvs.stop_rx) -> finish_tx => { + // collect the run time meta + self.sink(Arc::new(R::from(Runtime::load()?)))?; + finish_tx?.send(())?; + } + recv(self.recvs.recv) -> msg => { + self.sink(msg?)?; + } + } + + Ok(()) + } + + fn sink(&self, state: Arc) -> anyhow::Result<()> { + let mut file = self.file.lock().expect("failed to lock file"); + serde_json::to_writer(&mut *file, &state)?; + file.write_all(b",\n")?; + Ok(()) + } + + fn subscribe_data_type() -> RecordType { + RecordType::Data + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, time::Duration}; + + use consensus_engine::View; + + use crate::{ + network::{ + behaviour::NetworkBehaviour, + regions::{Region, RegionsData}, + Network, NetworkBehaviourKey, + }, + node::{ + dummy_streaming::{DummyStreamingNode, DummyStreamingState}, + Node, NodeId, NodeIdExt, + }, + output_processors::OutData, + runner::SimulationRunner, + warding::SimulationState, + }; + + use super::*; + #[derive(Debug, Clone, Serialize)] + struct SettingsRecord { + states: HashMap, + } + + impl TryFrom<&SimulationState> for SettingsRecord { + type Error = anyhow::Error; + + fn try_from(value: &SimulationState) -> Result { + Ok(Self { + states: value + .nodes + .read() + .iter() + .map(|node| (node.id(), node.current_view())) + .collect(), + }) + } + } + + #[test] + fn test_streaming() { + let simulation_settings = crate::settings::SimulationSettings { + seed: Some(1), + ..Default::default() + }; + + let nodes = (0..6) + .map(|idx| { + Box::new(DummyStreamingNode::new(NodeId::from_index(idx), ())) + as Box< + dyn Node + + std::marker::Send + + Sync, + > + }) + .collect::>(); + let network = Network::new( + RegionsData { + regions: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + (region, vec![NodeId::from_index(idx)]) + }) + .collect(), + node_region: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + (NodeId::from_index(idx), region) + }) + .collect(), + region_network_behaviour: (0..6) + .map(|idx| { + let region = match idx % 6 { + 0 => Region::Europe, + 1 => Region::NorthAmerica, + 2 => Region::SouthAmerica, + 3 => Region::Asia, + 4 => Region::Africa, + 5 => Region::Australia, + _ => unreachable!(), + }; + ( + NetworkBehaviourKey::new(region, region), + NetworkBehaviour { + delay: Duration::from_millis(100), + drop: 0.0, + }, + ) + }) + .collect(), + }, + 0, + ); + let simulation_runner: SimulationRunner<(), OutData, (), DummyStreamingState> = + SimulationRunner::new(network, nodes, Default::default(), simulation_settings).unwrap(); + simulation_runner.simulate().unwrap(); + } +} diff --git a/network-runner/src/warding/mod.rs b/network-runner/src/warding/mod.rs new file mode 100644 index 0000000..4241425 --- /dev/null +++ b/network-runner/src/warding/mod.rs @@ -0,0 +1,54 @@ +// std +use std::sync::Arc; +// crates +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; +// internal +use crate::runner::BoxedNode; + +mod ttf; + +pub struct SimulationState { + pub nodes: Arc>>>, +} + +impl SimulationState { + #[inline] + pub fn new(nodes: Vec>) -> Self { + Self { + nodes: Arc::new(RwLock::new(nodes)), + } + } +} + +/// A ward is a computation over the `NetworkState`, it must return true if the state satisfies +/// the warding conditions. It is used to stop the consensus simulation if such condition is reached. +pub trait SimulationWard { + type SimulationState; + fn analyze(&mut self, state: &Self::SimulationState) -> bool; +} + +/// Ward dispatcher +/// Enum to avoid Boxing (Box) wards. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Ward { + MaxView(ttf::MaxViewWard), +} + +impl Ward { + pub fn simulation_ward_mut( + &mut self, + ) -> &mut dyn SimulationWard> { + match self { + Ward::MaxView(ward) => ward, + } + } +} + +impl SimulationWard for Ward { + type SimulationState = SimulationState; + fn analyze(&mut self, state: &Self::SimulationState) -> bool { + self.simulation_ward_mut().analyze(state) + } +} diff --git a/network-runner/src/warding/ttf.rs b/network-runner/src/warding/ttf.rs new file mode 100644 index 0000000..9c242ee --- /dev/null +++ b/network-runner/src/warding/ttf.rs @@ -0,0 +1,41 @@ +use crate::warding::{SimulationState, SimulationWard}; +use serde::{Deserialize, Serialize}; + +/// Time to finality ward. It monitors the amount of rounds of the simulations, triggers when surpassing +/// the set threshold. +#[derive(Debug, Serialize, Deserialize, Copy, Clone)] +#[serde(transparent)] +pub struct MaxViewWard { + max_count: usize, +} + +impl SimulationWard for MaxViewWard { + type SimulationState = SimulationState; + fn analyze(&mut self, state: &Self::SimulationState) -> bool { + // state.nodes.read().iter(); + //.all(|n| n.current_view() >= self.max_count) + todo!() + } +} + +#[cfg(test)] +mod test { + use crate::warding::ttf::MaxViewWard; + use crate::warding::{SimulationState, SimulationWard}; + use parking_lot::RwLock; + use std::sync::Arc; + + #[test] + fn rebase_threshold() { + let mut ttf = MaxViewWard { max_count: 10 }; + + let node = 11; + let state = SimulationState { + nodes: Arc::new(RwLock::new(vec![Box::new(node)])), + }; + assert!(ttf.analyze(&state)); + + state.nodes.write().push(Box::new(9)); + assert!(!ttf.analyze(&state)); + } +}