commit 6ea3006af9645c92307e648a87b3401b0d2b8001 Author: Daniel Sanchez Quiros Date: Wed Oct 19 15:17:22 2022 +0200 Extract claro simulations into new repository diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a0038a --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +.idea \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..98dd1f2 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2052 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" + +[[package]] +name = "array-init-cursor" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" + +[[package]] +name = "arrow-format" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df5d25bc6d676271277120c41ef28760fe0a9f070677a58db621c0f983f9c20" +dependencies = [ + "planus", + "serde", +] + +[[package]] +name = "arrow2" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc54f0b14083abaf6bc71cf1aeccd7831a24b1e29d07683ba9a4a0f6c5d9326" +dependencies = [ + "ahash", + "arrow-format", + "base64", + "bytemuck", + "chrono", + "dyn-clone", + "either", + "fallible-streaming-iterator", + "foreign_vec", + "futures", + "hash_hasher", + "indexmap", + "json-deserializer", + "lexical-core", + "multiversion", + "num-traits", + "parquet2", + "simdutf8", + "streaming-iterator", + "strength_reduce", +] + +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitpacking" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" +dependencies = [ + "crunchy", +] + +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" + +[[package]] +name = "bytemuck" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5715e491b5a1598fc2bef5a606847b5dc1d48ea625bd3c02c00de8285591da" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9e1f5fa78f69496407a27ae9ed989e3c3b072310286f5ef385525e4cbc24a9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-integer", + "num-traits", + "time", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "clap" +version = "3.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap", + "once_cell", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_derive" +version = "3.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "claro" +version = "0.1.0" +dependencies = [ + "async-trait", + "rand", + "tokio", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "comfy-table" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b103d85ca6e209388771bfb7aa6b68a7aeec4afbf6f0a0264bfbf50360e5212e" +dependencies = [ + "crossterm", + "strum", + "strum_macros", + "unicode-width", +] + +[[package]] +name = "consensus-simulations" +version = "0.1.0" +dependencies = [ + "clap", + "claro", + "fixed-slice-deque", + "once_cell", + "polars", + "rand", + "rayon", + "serde", + "serde_json", + "snowball", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossterm" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2102ea4f781910f8a5b98dd061f4c2023f479ce7bb1236330099ceb5a93cf17" +dependencies = [ + "bitflags", + "crossterm_winapi", + "libc", + "mio", + "parking_lot", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +dependencies = [ + "winapi", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "csv-core" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +dependencies = [ + "memchr", +] + +[[package]] +name = "cxx" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f83d0ebf42c6eafb8d7c52f7e5f2d3003b89c7aa4fd2b79229209459a849af8" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07d050484b55975889284352b0ffc2ecbda25c0c55978017c132b29ba0818a86" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d2199b00553eda8012dfec8d3b1c75fce747cf27c169a270b3b99e3448ab78" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb67a6de1f602736dd7eaead0080cf3435df806c61b24b13328db128c58868f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dyn-clone" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f94fa09c2aeea5b8839e414b7b841bf429fd25b9c522116ac97ee87856d88b2" + +[[package]] +name = "either" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fixed-slice-deque" +version = "0.1.0-beta1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37163275a02523b0589dae3557885fe34062f194253340b59b6e4c2e9f88cbbf" +dependencies = [ + "slice-deque", +] + +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "foreign_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1b05cbd864bcaecbd3455d6d967862d446e4ebfc3c2e5e5b9841e53cba6673" + +[[package]] +name = "futures" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf" + +[[package]] +name = "futures-executor" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68" + +[[package]] +name = "futures-macro" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56" + +[[package]] +name = "futures-task" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" + +[[package]] +name = "futures-util" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "hash_hasher" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", + "rayon", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "indexmap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +dependencies = [ + "async-trait", + "futures-util", +] + +[[package]] +name = "itoa" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" + +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "json-deserializer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47631885425c482fcf2dc4b182fc973c3c5b81a8f43a028055559bd24cccfa6e" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6" +dependencies = [ + "lexical-core", +] + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.135" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" + +[[package]] +name = "libm" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565" + +[[package]] +name = "link-cplusplus" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" +dependencies = [ + "cc", +] + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "lz4" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +dependencies = [ + "libc", + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memmap2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95af15f345b17af2efc8ead6080fb8bc376f8cec1b35277b935637595fe77498" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.36.1", +] + +[[package]] +name = "multiversion" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373" +dependencies = [ + "multiversion-macros", +] + +[[package]] +name = "multiversion-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" + +[[package]] +name = "os_str_bytes" +version = "6.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.42.0", +] + +[[package]] +name = "parquet-format-async-temp" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1a672c84c3e5b5eb6530286b2d22cc1ea8e1e3560e4c314218d6ab749c6db99" +dependencies = [ + "async-trait", + "futures", + "integer-encoding", +] + +[[package]] +name = "parquet2" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa7afeef767fb16a37216cdf906268ae10db9825aa1ed8723b8493b1e99b0b" +dependencies = [ + "async-stream", + "bitpacking", + "brotli", + "flate2", + "futures", + "lz4", + "parquet-format-async-temp", + "snap", + "streaming-decompression", + "zstd", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "planus" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1691dd09e82f428ce8d6310bd6d5da2557c82ff17694d2a32cad7242aea89f" +dependencies = [ + "array-init-cursor", +] + +[[package]] +name = "polars" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75b1077fda63c0f67acc1cdc8586e7afce419be1e85bf7dfa8935e0e266d6b3" +dependencies = [ + "polars-core", + "polars-io", + "polars-lazy", + "polars-ops", + "polars-time", +] + +[[package]] +name = "polars-arrow" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b28f858b252436550679609a23be34d62705faf783887f172f845eb58bcb8b" +dependencies = [ + "arrow2", + "hashbrown", + "num", + "thiserror", +] + +[[package]] +name = "polars-core" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaec1ca3ac4829ca24b33743adeeb323a43b5a85515bfce20c2c81799c82790" +dependencies = [ + "ahash", + "anyhow", + "arrow2", + "bitflags", + "chrono", + "comfy-table", + "hashbrown", + "indexmap", + "num", + "once_cell", + "polars-arrow", + "polars-utils", + "rand", + "rand_distr", + "rayon", + "regex", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "polars-io" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51405e46f93e306a3c9280c60ba1101c662e8a6dab33344680d31c3161045f1c" +dependencies = [ + "ahash", + "anyhow", + "arrow2", + "csv-core", + "dirs", + "lexical", + "lexical-core", + "memchr", + "memmap2", + "num", + "once_cell", + "polars-arrow", + "polars-core", + "polars-time", + "polars-utils", + "rayon", + "regex", + "serde_json", + "simdutf8", +] + +[[package]] +name = "polars-lazy" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340af778bc8124180d8ca1a566f076a5339566a207a42130796048b087fe977" +dependencies = [ + "ahash", + "bitflags", + "glob", + "parking_lot", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-time", + "polars-utils", + "rayon", +] + +[[package]] +name = "polars-ops" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1812e5d5e589d5bd23f8d89dcd8bd4508082c50d055b8ff5fafb6f2a519c9a" +dependencies = [ + "polars-arrow", + "polars-core", +] + +[[package]] +name = "polars-time" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4ebe97d601a4b443337df71d0b7e673fce953654871c3311850ea394d48297" +dependencies = [ + "chrono", + "lexical", + "polars-arrow", + "polars-core", + "polars-utils", +] + +[[package]] +name = "polars-utils" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea836afadcddee3f1a513dae7624f6d7d0d64abb129063ec7476b8347c8725b" +dependencies = [ + "parking_lot", + "rayon", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "rayon" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" + +[[package]] +name = "rustversion" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" + +[[package]] +name = "ryu" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scratch" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" + +[[package]] +name = "serde" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "slice-deque" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31ef6ee280cdefba6d2d0b4b78a84a1c1a3f3a4cec98c2d4231c8bc225de0f25" +dependencies = [ + "libc", + "mach", + "winapi", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "snap" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" + +[[package]] +name = "snowball" +version = "0.1.0" +dependencies = [ + "claro", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "streaming-decompression" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6cc3b19bfb128a8ad11026086e31d3ce9ad23f8ea37354b31383a187c44cf3" +dependencies = [ + "fallible-streaming-iterator", +] + +[[package]] +name = "streaming-iterator" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55dd09aaa2f85ef8767cc9177294d63c30d62c8533329e75aa51d8b94976e22" + +[[package]] +name = "strength_reduce" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ff2f71c82567c565ba4b3009a9350a96a7269eaa4001ebedae926230bc2254" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" + +[[package]] +name = "strum_macros" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "tokio" +version = "1.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +dependencies = [ + "autocfg", + "num_cpus", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "nu-ansi-term", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" + +[[package]] +name = "unicode-segmentation" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" +dependencies = [ + "cc", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..aedf283 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[workspace] + +members = [ + "consensus/claro", + "consensus/snowball", + "simulations/snow-family" +] + +[profile.release-opt] +inherits = "release" +lto = true \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2fe4b04 --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# Consensus Research + +## Project Structure + +* `consensus`: Consensus implementation libraries + * `snowball`: Snowball implementation + * `claro`: Claro implementation +* `prototypes`: Simulations and experiments related libraries and binaries + * `consensus-simulations`: Consensus simulations app + +## Build & Test + +Minimal Rust supported version: `1.63` + +When in development, please, use `cargo clippy` to build the project. Any warning is promoted to an error in our CI. + +* Use `cargo test` for executing tests, and `cargo test -- --nocapture` for seeing test outputs. +* Use `cargo run --exampel {example_name}` to run an example. + +### Build Documentation + +Simply run `cargo doc --open --no-deps` to build and access a copy of the generated documentation. diff --git a/consensus/claro/Cargo.toml b/consensus/claro/Cargo.toml new file mode 100644 index 0000000..48a9310 --- /dev/null +++ b/consensus/claro/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "claro" +version = "0.1.0" +edition = "2021" +authors = [ + "Daniel Sanchez Quiros " +] +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = "0.1" +rand = "0.8" +tracing = "0.1" +tracing-core = "0.1" +tracing-subscriber = { version = "0.3", features = ["fmt", "json", "std"] } +tokio = { version = "1.17", features = ["sync"] } + +[dev-dependencies] +tokio = { version = "1.17", features = ["rt-multi-thread", "macros"] } + +[features] +default = [] +testing = [] \ No newline at end of file diff --git a/consensus/claro/src/claro.rs b/consensus/claro/src/claro.rs new file mode 100644 index 0000000..cf8dc55 --- /dev/null +++ b/consensus/claro/src/claro.rs @@ -0,0 +1,361 @@ +// std +use std::fmt::{Debug, Display, Formatter}; +use std::marker::PhantomData; +use tracing::debug; +// crates +// internal +use crate::query::NodeQuery; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum Vote { + Yes(Tx), + No(Tx), +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum Opinion { + None(Tx), + Yes(Tx), + No(Tx), +} + +impl Opinion { + pub fn flip(self) -> Self { + match self { + Opinion::Yes(tx) => Opinion::No(tx), + Opinion::No(tx) => Opinion::Yes(tx), + none => none, + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum Decision { + Decided(Opinion), + Undecided(Opinion), +} + +impl Display for Opinion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let tag = match self { + Opinion::Yes(_) => "yes", + Opinion::No(_) => "no", + Opinion::None(_) => "none", + }; + write!(f, "{}", tag) + } +} + +impl Display for Decision { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let tag = match self { + Decision::Decided(_) => "decided", + Decision::Undecided(_) => "undecided", + }; + write!(f, "{}", tag) + } +} + +impl From> for Option> { + fn from(opinion: Opinion) -> Self { + match opinion { + Opinion::Yes(tx) => Some(Vote::Yes(tx)), + Opinion::No(tx) => Some(Vote::No(tx)), + Opinion::None(_) => None, + } + } +} + +impl From> for Opinion { + fn from(vote: Vote) -> Self { + match vote { + Vote::Yes(tx) => Opinion::Yes(tx), + Vote::No(tx) => Opinion::No(tx), + } + } +} + +impl From> for Option> { + fn from(decision: Decision) -> Self { + match decision { + Decision::Decided(opinion) | Decision::Undecided(opinion) => opinion.into(), + } + } +} + +#[allow(dead_code)] +/// Claro round computed evidence, confidence and alpha +pub struct ClaroRoundCalculation { + confidence: f32, + e1: f32, + e2: f32, + e: f32, + alpha: f32, +} + +/// Claro internal state +#[derive(Default, Debug)] +pub struct ClaroState { + /// Positive votes seen + evidence: usize, + /// Total votes seen, positive and negative + evidence_accumulated: usize, + /// Votes ratio + confidence: usize, +} + +impl ClaroState { + pub fn update_confidence(&mut self, votes: &[Vote]) { + let total_votes = votes.len(); + self.confidence = self.confidence.saturating_add(total_votes); + } + + pub fn update_evidence(&mut self, votes: &[Vote]) { + let total_votes = votes.len(); + let total_yes = votes.iter().filter(|v| matches!(v, Vote::Yes(_))).count(); + self.evidence = self.evidence.saturating_add(total_yes); + self.evidence_accumulated = self.evidence_accumulated.saturating_add(total_votes); + } + + pub fn confidence(&self) -> usize { + self.confidence + } + + pub fn evidence(&self) -> usize { + self.evidence + } + + pub fn evidence_accumulated(&self) -> usize { + self.evidence_accumulated + } +} + +/// Node query configuration +#[derive(Debug, Clone, Copy)] +pub struct QueryConfiguration { + /// How many nodes to query + pub query_size: usize, + /// Initial query + pub initial_query_size: usize, + /// Growth increment per claro round + pub query_multiplier: usize, + /// Max value for [`QueryConfiguration::query_multiplier`] + pub max_multiplier: usize, +} + +impl QueryConfiguration { + #[allow(dead_code)] + pub fn new(query_size: usize) -> Self { + Self { + query_size, + initial_query_size: query_size, + // TODO: Should this be configurable? Runtime vs Compiled + query_multiplier: 2, + max_multiplier: 4, + } + } + + /// Increment query based upon configuration + /// query_size = min(query_size * growth_constant, initial_query_size * growth_max) + fn grow(&mut self) { + self.query_size = (self.query_size * self.query_multiplier) + .min(self.initial_query_size * self.max_multiplier); + } +} + +/// Claro algorithm configuration +#[derive(Debug, Clone, Copy)] +pub struct ClaroConfiguration { + pub evidence_alpha: f32, + pub evidence_alpha_2: f32, + pub confidence_beta: f32, + pub look_ahead: usize, + pub query: QueryConfiguration, +} + +/// Claro computation object +pub struct ClaroSolver { + _phantom: PhantomData, + /// Internal state + state: ClaroState, + /// Configuration, including node query configuration + configuration: ClaroConfiguration, + /// Current tx decision + decision: Decision, + /// Node query setup for current node + node_query: NodeQuery, +} + +// TODO: can we remove clone here? +impl ClaroSolver { + pub fn new(tx: Tx, configuration: ClaroConfiguration, node_query: NodeQuery) -> Self { + Self { + _phantom: Default::default(), + state: Default::default(), + decision: Decision::Undecided(Opinion::Yes(tx)), + configuration, + node_query, + } + } + + pub fn with_initial_opinion( + configuration: ClaroConfiguration, + node_query: NodeQuery, + opinion: Opinion, + ) -> Self { + Self { + _phantom: Default::default(), + state: Default::default(), + decision: Decision::Undecided(opinion), + configuration, + node_query, + } + } + + /// Compute a single round state from already queried nodes votes + fn round_state(&self, votes: &[Vote]) -> ClaroRoundCalculation { + let total_votes = votes.len(); + let yes_votes = votes.iter().filter(|&v| matches!(v, Vote::Yes(_))).count(); + let confidence = self.state.confidence() as f32 + / (self.state.confidence() as f32 + self.configuration.look_ahead as f32); + + let e1 = yes_votes as f32 / total_votes as f32; + let e2 = self.state.evidence() as f32 / self.state.evidence_accumulated() as f32; + let e = e1 * (1f32 - confidence) + e2 * confidence; + let alpha = self.configuration.evidence_alpha * (1f32 - confidence) + + self.configuration.evidence_alpha_2 * confidence; + + ClaroRoundCalculation { + confidence, + e1, + e2, + e, + alpha, + } + } + + /// Compute a single round + /// mutates the decision parameter upon this round data + pub fn step(&mut self, tx: Tx, votes: &[Vote]) { + assert!(matches!(self.decision, Decision::Undecided(_))); + debug!(votes = ?votes); + if let Decision::Undecided(Opinion::None(_)) = self.decision() { + if let Some(vote) = votes.first().cloned() { + self.decision = Decision::Undecided(vote.into()); + } + } + + if !votes.is_empty() { + self.state.update_evidence(votes); + self.state.update_confidence(votes); + + let ClaroRoundCalculation { + e, + alpha, + confidence, + .. + } = self.round_state(votes); + debug!(e = e, alpha = alpha); + if e > alpha { + self.decision = Decision::Undecided(Opinion::Yes(tx)); + } else if e < 1f32 - alpha { + self.decision = Decision::Undecided(Opinion::No(tx)); + } else { + self.configuration.query.grow(); + } + if confidence > self.configuration.confidence_beta { + self.decision = Decision::Decided(self.opinion()); + } + } + } + + /// Derive vote from it's current decision + pub fn vote(&self) -> Option> { + self.decision.clone().into() + } + + pub fn decision(&self) -> Decision { + self.decision.clone() + } + + pub fn opinion(&self) -> Opinion { + match &self.decision { + Decision::Decided(o) | Decision::Undecided(o) => o.clone(), + } + } + + pub fn state(&self) -> &ClaroState { + &self.state + } + + pub fn node_query(&self) -> &NodeQuery { + &self.node_query + } +} + +#[cfg(test)] +mod test { + use crate::claro::{Decision, ClaroConfiguration, ClaroSolver, QueryConfiguration, Vote}; + use crate::query::NodeQuery; + use crate::testing::query::*; + use crate::{Opinion, VoteQuery}; + use std::fmt::Debug; + + #[derive(Clone, Eq, PartialEq, Debug)] + struct EmptyTx; + + fn test_all_votes( + tx: Tx, + votes: &[Vote], + expected: Decision, + ) { + let config = ClaroConfiguration { + evidence_alpha: 0.01, + evidence_alpha_2: 0.01, + confidence_beta: 0.01, + look_ahead: 1, + query: QueryConfiguration::new(10), + }; + let node_query = NodeQuery::new(config.query.query_size, "node_1".into()); + let mut solver = ClaroSolver::new(tx.clone(), config, node_query); + + assert_eq!( + solver.decision, + Decision::Undecided(Opinion::Yes(tx.clone())) + ); + solver.step(tx, votes); + assert_eq!(solver.decision, expected); + } + + #[test] + fn all_approved() { + let votes: Vec<_> = (0..10).map(|_| Vote::::Yes(true)).collect(); + test_all_votes::(true, &votes, Decision::Decided(Opinion::Yes(true))); + } + + #[test] + fn all_rejected() { + let votes: Vec<_> = (0..10).map(|_| Vote::::No(true)).collect(); + test_all_votes::(true, &votes, Decision::Decided(Opinion::No(true))); + } + + #[tokio::test] + async fn loop_all_approved() { + let vote = Vote::Yes(EmptyTx); + let mut fixed_query = FixedQuery::new(vote.clone()); + let config = ClaroConfiguration { + evidence_alpha: 0.01, + evidence_alpha_2: 0.01, + confidence_beta: 0.01, + look_ahead: 1, + query: QueryConfiguration::new(10), + }; + + let node_query = NodeQuery::new(config.query.query_size, "node_1".into()); + let mut solver = ClaroSolver::new(EmptyTx, config, node_query); + + let query = fixed_query.query(&solver.node_query, EmptyTx).await; + solver.step(EmptyTx, &query); + assert_eq!(solver.vote(), Some(vote)) + } +} diff --git a/consensus/claro/src/lib.rs b/consensus/claro/src/lib.rs new file mode 100644 index 0000000..bc053be --- /dev/null +++ b/consensus/claro/src/lib.rs @@ -0,0 +1,12 @@ +mod claro; +mod query; +mod tracing; + +#[cfg(feature = "testing")] +pub mod testing; + +pub use self::claro::{ + Decision, ClaroConfiguration, ClaroSolver, ClaroState, Opinion, QueryConfiguration, Vote, +}; +pub use self::query::{NodeId, NodeQuery, NodeWeight, NodesSample, VoteQuery}; +pub use self::tracing::{claro_tracing_layer_with_writer, CLARO_TARGET_TAG}; diff --git a/consensus/claro/src/query.rs b/consensus/claro/src/query.rs new file mode 100644 index 0000000..d7dcc36 --- /dev/null +++ b/consensus/claro/src/query.rs @@ -0,0 +1,107 @@ +use crate::claro::Vote; +use rand::seq::SliceRandom; +use rand::thread_rng; +use std::collections::HashMap; +use tracing::debug; + +// TODO: Check on proper types +/// Node ids type +pub type NodeId = String; +/// Node weight alias +/// Refers to amount of staking a node holds +pub type NodeWeight = f64; + +/// Node ids <=> weights sampling information trait +pub trait NodesSample { + fn nodes(&self) -> Vec; + fn weights(&self) -> HashMap<&NodeId, NodeWeight>; +} + +/// Selector of nodes, random sample for some size `K` +#[derive(Debug, Clone)] +pub struct NodeQuery { + node_size: usize, + node_id: NodeId, +} + +impl NodeQuery { + pub fn new(node_size: usize, node_id: NodeId) -> Self { + Self { node_size, node_id } + } + + pub fn query_size(&self) -> usize { + self.node_size + } + + pub fn node_id(&self) -> &NodeId { + &self.node_id + } + + pub fn sample(&self, node_sample: &Sample) -> Vec { + let node_ids = node_sample.nodes(); + let weights = node_sample.weights(); + // TODO: do we need to be reproducible? + let mut rng = thread_rng(); + let node_ids = node_ids + .as_slice() + .choose_multiple_weighted(&mut rng, self.node_size + 1, |e| *weights.get(e).unwrap()) + .unwrap() + .cloned() + .filter(|node_id| node_id != &self.node_id) + .take(self.node_size) + .collect(); + debug!(query_node_ids = ?node_ids); + node_ids + } +} + +/// Communication layer abstraction trait +/// Used by the claro algorithm runner to query for the votes of other nodes +#[async_trait::async_trait] +pub trait VoteQuery: Send + Sync { + type Tx; + async fn query(&mut self, node_query: &NodeQuery, tx: Self::Tx) -> Vec>; +} + +#[cfg(test)] +mod test { + use crate::query::{NodeId, NodeQuery, NodeWeight, NodesSample}; + use std::collections::{HashMap, HashSet}; + + struct TestSample { + node_ids: Vec, + node_weights: Vec, + } + + impl TestSample { + fn len(&self) -> usize { + assert_eq!(self.node_weights.len(), self.node_ids.len()); + self.node_ids.len() + } + } + + impl NodesSample for TestSample { + fn nodes(&self) -> Vec { + self.node_ids.clone() + } + + fn weights(&self) -> HashMap<&NodeId, NodeWeight> { + self.node_ids + .iter() + .zip(self.node_weights.iter().copied()) + .collect() + } + } + + #[test] + fn unique_sample_set() { + let query: NodeQuery = NodeQuery::new(10, "".into()); + let sample = TestSample { + node_ids: (0..10).map(|i| i.to_string()).collect(), + node_weights: (1..11usize).map(|i| i as f64).collect(), + }; + + let ids: HashSet<_> = query.sample(&sample).into_iter().collect(); + assert_eq!(ids.len(), sample.len()); + } +} diff --git a/consensus/claro/src/testing/mod.rs b/consensus/claro/src/testing/mod.rs new file mode 100644 index 0000000..67350db --- /dev/null +++ b/consensus/claro/src/testing/mod.rs @@ -0,0 +1 @@ +pub mod query; diff --git a/consensus/claro/src/testing/query.rs b/consensus/claro/src/testing/query.rs new file mode 100644 index 0000000..02f32b1 --- /dev/null +++ b/consensus/claro/src/testing/query.rs @@ -0,0 +1,38 @@ +use crate::{NodeQuery, Vote, VoteQuery}; +use std::marker::PhantomData; + +pub struct NoQuery(PhantomData); + +impl Default for NoQuery { + fn default() -> Self { + Self(Default::default()) + } +} + +#[async_trait::async_trait] +impl VoteQuery for NoQuery { + type Tx = Tx; + + async fn query(&mut self, _node_query: &NodeQuery, _tx: Self::Tx) -> Vec> { + vec![] + } +} + +pub struct FixedQuery(Vote); + +impl FixedQuery { + pub fn new(vote: Vote) -> Self { + Self(vote) + } +} + +#[async_trait::async_trait] +impl VoteQuery for FixedQuery { + type Tx = Tx; + + async fn query(&mut self, node_query: &NodeQuery, _tx: Self::Tx) -> Vec> { + (0..node_query.query_size()) + .map(|_| self.0.clone()) + .collect() + } +} diff --git a/consensus/claro/src/tracing.rs b/consensus/claro/src/tracing.rs new file mode 100644 index 0000000..244cf91 --- /dev/null +++ b/consensus/claro/src/tracing.rs @@ -0,0 +1,17 @@ +use tracing_core::Subscriber; +use tracing_subscriber::filter::filter_fn; +use tracing_subscriber::fmt::MakeWriter; +use tracing_subscriber::Layer; + +pub const CLARO_TARGET_TAG: &str = "CLARO_TARGET"; + +pub fn claro_tracing_layer_with_writer(writer: W, filter_tag: &'static str) -> impl Layer +where + W: for<'writer> MakeWriter<'writer> + 'static, + S: Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>, +{ + tracing_subscriber::fmt::layer() + .with_writer(writer) + .json() + .with_filter(filter_fn(move |metadata| metadata.target() == filter_tag)) +} diff --git a/consensus/snowball/Cargo.toml b/consensus/snowball/Cargo.toml new file mode 100644 index 0000000..091f4b0 --- /dev/null +++ b/consensus/snowball/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "snowball" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +claro = { path = "../claro" } + +[dev-dependencies] +claro = { path = "../claro", features = ["testing"]} \ No newline at end of file diff --git a/consensus/snowball/src/lib.rs b/consensus/snowball/src/lib.rs new file mode 100644 index 0000000..9049e70 --- /dev/null +++ b/consensus/snowball/src/lib.rs @@ -0,0 +1,7 @@ +#[allow(dead_code)] +mod snowball; + +pub use crate::snowball::{SnowballConfiguration, SnowballSolver}; + +/// Snowball logging filtering tag +pub const SNOWBALL_TARGET_TAG: &str = "SNOWBALL_TARGET"; diff --git a/consensus/snowball/src/snowball.rs b/consensus/snowball/src/snowball.rs new file mode 100644 index 0000000..e2fbef7 --- /dev/null +++ b/consensus/snowball/src/snowball.rs @@ -0,0 +1,172 @@ +use claro::{Decision, NodeQuery, Opinion, Vote}; +use std::fmt::Debug; + +/// Snowball algorithm configuration +#[derive(Debug, Clone, Copy)] +pub struct SnowballConfiguration { + pub quorum_size: usize, + pub sample_size: usize, + pub decision_threshold: usize, +} + +/// Snowball computation object +pub struct SnowballSolver { + configuration: SnowballConfiguration, + decision: Decision, + consecutive_success: u64, + node_query: NodeQuery, +} + +impl SnowballSolver { + pub fn new(tx: Tx, configuration: SnowballConfiguration, node_query: NodeQuery) -> Self { + Self { + configuration, + decision: Decision::Undecided(Opinion::None(tx)), + consecutive_success: 0, + node_query, + } + } + + pub fn with_initial_opinion( + configuration: SnowballConfiguration, + node_query: NodeQuery, + opinion: Opinion, + ) -> Self { + Self { + configuration, + decision: Decision::Undecided(opinion), + consecutive_success: 0, + node_query, + } + } + + fn count_opinion_votes(&self, votes: &[Vote]) -> usize { + votes + .iter() + .filter(|v| { + matches!( + (v, self.vote()), + (Vote::Yes(_), Some(Vote::Yes(_))) | (Vote::No(_), Some(Vote::No(_))) + ) + }) + .count() + } + + pub fn step(&mut self, votes: &[Vote]) { + assert!(matches!(self.decision, Decision::Undecided(_))); + + let preference_count = self.count_opinion_votes(votes); + let not_preference_count = votes.len() - preference_count; + + if preference_count >= self.configuration.quorum_size { + self.consecutive_success += 1; + } else if not_preference_count >= self.configuration.quorum_size { + self.decision = Decision::Undecided(self.opinion().flip()); + self.consecutive_success = 1; + } else { + self.consecutive_success = 0 + } + + if self.consecutive_success > self.configuration.decision_threshold as u64 { + self.decision = Decision::Decided(self.opinion()) + } + } + + pub fn consecutive_success(&self) -> u64 { + self.consecutive_success + } + + pub fn decision(&self) -> Decision { + self.decision.clone() + } + + pub fn opinion(&self) -> Opinion { + match &self.decision { + Decision::Decided(o) | Decision::Undecided(o) => o.clone(), + } + } + + /// Derive vote from it's current decision + pub fn vote(&self) -> Option> { + self.decision().into() + } + + pub fn node_query(&self) -> &NodeQuery { + &self.node_query + } +} + +#[cfg(test)] +mod test { + use super::{SnowballConfiguration, SnowballSolver}; + use claro::{Decision, NodeQuery, Opinion, Vote}; + + #[test] + fn test_change_opinion() { + let configuration = SnowballConfiguration { + quorum_size: 1, + sample_size: 10, + decision_threshold: 10, + }; + + let mut solver = SnowballSolver::with_initial_opinion( + configuration, + NodeQuery::new(0, "0".to_string()), + Opinion::Yes(true), + ); + + let votes = vec![Vote::No(true); 10]; + solver.step(&votes); + assert!(matches!(solver.decision(), Decision::Undecided(_))); + assert_eq!(solver.consecutive_success, 1); + assert_eq!(solver.opinion(), Opinion::No(true)); + } + + #[test] + fn test_makes_decision() { + let configuration = SnowballConfiguration { + quorum_size: 1, + sample_size: 10, + decision_threshold: 10, + }; + let beta = configuration.decision_threshold; + + let mut solver = SnowballSolver::with_initial_opinion( + configuration, + NodeQuery::new(0, "0".to_string()), + Opinion::Yes(true), + ); + + let votes = vec![Vote::No(true); 10]; + for _ in 0..beta + 1 { + solver.step(&votes); + } + + assert_eq!(solver.decision(), Decision::Decided(Opinion::No(true))); + assert_eq!(solver.consecutive_success, beta as u64 + 1); + assert_eq!(solver.opinion(), Opinion::No(true)); + } + + #[test] + fn test_reset_consecutive_counter() { + let configuration = SnowballConfiguration { + quorum_size: 2, + sample_size: 10, + decision_threshold: 10, + }; + + let mut solver = SnowballSolver::with_initial_opinion( + configuration, + NodeQuery::new(0, "0".to_string()), + Opinion::Yes(true), + ); + + let votes = vec![Vote::No(true), Vote::Yes(true)]; + + solver.step(&votes); + + assert_eq!(solver.consecutive_success, 0); + assert_eq!(solver.opinion(), Opinion::Yes(true)); + assert!(matches!(solver.decision(), Decision::Undecided(_))); + } +} diff --git a/simulations/snow-family/Cargo.toml b/simulations/snow-family/Cargo.toml new file mode 100644 index 0000000..93b9f8f --- /dev/null +++ b/simulations/snow-family/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "consensus-simulations" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = { version = "3.2", features = ["derive"] } +claro = { path = "../../consensus/claro", features = ["testing"] } +once_cell = "1.13" +polars = { version = "0.23", features = ["serde", "object", "json", "csv-file", "parquet", "dtype-struct"] } +rand = { version = "0.8", features = ["small_rng"] } +rayon = "1.5" +fixed-slice-deque = "0.1.0-beta1" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +snowball = { path = "../../consensus/snowball" } diff --git a/simulations/snow-family/README.md b/simulations/snow-family/README.md new file mode 100644 index 0000000..b6cd6bb --- /dev/null +++ b/simulations/snow-family/README.md @@ -0,0 +1,323 @@ +# Consensus simulations + + +## Usage + +### Build + +Run cargo build command under the general project folder (consensus-prototypes): + +```shell +cargo build --profile release-opt --bin snow-family +``` + +Built binary is usually placed at `target/release-opt/consensus-simulations`, +or if built for a specific architecture (overridden) at `target/{ARCH}/release-opt/consensus-simulations`. + +### Execute + +Move binary at some place of your choice, or after build run (from the main project folder): + +```shell +./target/release-opt/snow-family --help +``` + +``` +consensus-simulations +Main simulation wrapper Pipes together the cli arguments with the execution + +USAGE: + snow-family.exe [OPTIONS] --input-settings --output-file + +OPTIONS: + -f, --output-format Output format selector [default: parquet] + -h, --help Print help information + -i, --input-settings Json file path, on `SimulationSettings` format + -o, --output-file Output file path + +``` + +## SimulationSettings + +Simulations are configured with a `json` settings description file like in the example: + +```json +{ + "consensus_settings": { + "snow_ball": { + "quorum_size": 14, + "sample_size": 20, + "decision_threshold": 20 + } + }, + "distribution": { + "yes": 0.6, + "no": 0.4, + "none": 0.0 + }, + "byzantine_settings": { + "total_size": 10000, + "distribution": { + "honest": 1.0, + "infantile": 0.0, + "random": 0.0, + "omniscient": 0.0 + } + }, + "wards": [ + { + "time_to_finality": { + "ttf_threshold" : 1 + } + } + ], + "network_modifiers": [ + { + "random_drop": { + "drop_rate": 0.01 + } + } + ], + "seed" : 18042022 +} +``` + +### consensus_settings + +`consensus_settings` is the consensus backend configuration, the following consensus are supported: + +* [`snow_ball`](#Snowball) +* [`claro`](#Claro) + +#### Snowball + +Attributes: + +* `quorum_size`: `usize`, `alpha` as per the snowball algorithm +* `sample_size`: `usize`, `K` as per the snowball algorithm +* `decision_threshold`: `usize`, `beta` as per the snowball algorithm + +Example: + +```json +{ + "quorum_size": 14, + "sample_size": 20, + "decision_threshold": 20 +} +``` + +#### Claro + +Attributes: + +* `evidence_alpha`: `f32`, `alpha` as per the claro algorithm +* `evidence_alpha_2`: `f32`, `alpha2` as per the claro algorithm +* `confidence_beta`: `f32`, `beta` as per the claro algorithm (AKA decision threshold) +* `look_ahead`: `usize`, `l` as per the claro algorithm +* `query`: `QueryConfiguration`: + * `query_size`: `usize`, step node query size + * `initial_query_size`: `usize`, base query size (usually same as `query_size`) + * `query_multiplier`: `usize`, query size calculation in case no quorum found in step query + * `max_multiplier`: `usize`, max query multiplier to apply + +Example: + +```json +{ + "evidence_alpha": 0.8, + "evidence_alpha_2": 0.5, + "confidence_beta": 0.8, + "look_ahead": 20, + "query": { + "query_size": 30, + "initial_query_size": 30, + "query_multiplier": 2, + "max_multiplier": 4 + } +} +``` + +### distribution + +Initial **honest nodes** opinion distribution (**normalized**, must sum up to `1.0`) + +* `yes`, initial yes distribution +* `no`, initial no distribution +* `none`, initial none opinion distribution + +Example: + +```json +{ + "yes": 0.6, + "no": 0.4, + "none": 0.0 +} +``` + + +### byzantine_settings + +Byzantine nodes configuration + +* `total_size`: `usize`, total number of nodes to be spawned +* `distribution`: **normalized** distribution on hones/byzantine nodes + * `honest`: `f32`, **normalized** amount of hones nodes + * `infantile`: `f32`, **normalized** amount of infantile nodes + * `random`: `f32`, **normalized** amount of random nodes + * `omniscient`: `f32`, **normalized** amount of omniscient nodes + +Example: + +```json +{ + "total_size": 10000, + "distribution": { + "honest": 1.0, + "infantile": 0.0, + "random": 0.0, + "omniscient": 0.0 + } +} +``` + +### Simulation style + +Simulation can be done in different fashions: + +* *Sync*, (**default**) nodes run per step at the same time, updating on the previous states. +* *Async*, nodes run per batches (*chunks*) of predefined sizes +* *Glauber*, use the [glauber symulations solver](https://en.wikipedia.org/wiki/Glauber_dynamics) + * `update_rate`, record network state every `update_rate` processed chunks. + * `maximum_iterations`, threshold limit of simulation iterations + +Example: + +```json +{ + ..., + "simulation_style": "Sync" +} +``` + +```json +{ + ..., + "simulation_style": { + "Async" : { + "chunks": 20 + } + } +} +``` + +```json +{ + ..., + "simulation_style": { + "Glauber" : { + "update_rate": 1000, + "maximum_iterations":1000000 + } + } +} +``` + +### wards + +List of configurable experiment stop conditions based on the network state. + +* `time_to_finality`, break when reaching a threshold number of consensus rounds + * `ttf_threshold`: `usize`, threshold to be rebased + + + +```json +[ + { + "time_to_finality": { + "ttf_threshold" : 1 + } + } +] +``` + +* `stabilised`, break when for `n` rounds the network state keeps the same + * `buffer`: `usize`, consecutive number of rounds or iterations to check + * `check`: selector of what the ward should be aware of for checking states: + * `rounds`: check by consecutive rounds + * `iterations`: check every `n` iterations + +```json +[ + { + "stabilised": { + "buffer" : 3, + "check" : { "type": "rounds" } + } + } +] +``` + +or + +```json +[ + { + "stabilised": { + "buffer" : 3, + "check" : { "type": "iterations", "chunk": 100 } + } + } +] +``` + +* `converged`, break when a specified ratio of decided nodes is reached + * `ratio`, `[0.0-1.0]` range of decided nodes threshold + +### network_modifiers + +List of modifiers that handle the network state in each step iteration + +* `random_drop`, drop a percentage of the votes (setting them up as `None`) + * `drop_rate`: `f32`, normalize rate of dropped messages + +Example: + +```json +[ + { + "random_drop": { + "drop_rate": 0.01 + } + } +] +``` + +### seed + +The simulations can be run with a customized seed (otherwise is provided by the app itself) in order to make reproducible +runs. An `u64` integer must be provided + +```json +{ + ... + "seed" : 18042022 +} +``` + +## Output format + +Output is a [`Polars::Dataframe`](https://docs.rs/polars/latest/polars/frame/struct.DataFrame.html) [python version](https://pola-rs.github.io/polars/py-polars/html/reference/api/polars.DataFrame.html) + +Columns are vote states for each round (from `0`, initial state, to experiment end round). + +Three modes are supported, `["json", "csv", "parquet"]`, all of them standard dumps of `polars`. + +### Votes + +Votes are encoded as: +* `None` => `0` +* `Yes` => `1` +* `No` => `2` \ No newline at end of file diff --git a/simulations/snow-family/src/app.rs b/simulations/snow-family/src/app.rs new file mode 100644 index 0000000..358a071 --- /dev/null +++ b/simulations/snow-family/src/app.rs @@ -0,0 +1,150 @@ +// std +use std::error::Error; +use std::fmt::{Display, Formatter}; +use std::fs::File; +use std::io::Cursor; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +// crates +use crate::output_processors::OutData; +use clap::Parser; +use polars::io::SerWriter; +use polars::prelude::{DataFrame, JsonReader, SerReader}; +use serde::de::DeserializeOwned; +// internal +use crate::runner::SimulationRunner; +use crate::settings::SimulationSettings; + +/// Output format selector enum +#[derive(Debug, Default)] +enum OutputFormat { + Json, + Csv, + #[default] + Parquet, +} + +impl Display for OutputFormat { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let tag = match self { + OutputFormat::Json => "json", + OutputFormat::Csv => "csv", + OutputFormat::Parquet => "parquet", + }; + write!(f, "{}", tag) + } +} + +impl FromStr for OutputFormat { + type Err = std::io::Error; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "json" => Ok(Self::Json), + "csv" => Ok(Self::Csv), + "parquet" => Ok(Self::Parquet), + tag => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "Invalid {} tag, only [json, csv, polars] are supported", + tag + ), + )), + } + } +} + +/// Main simulation wrapper +/// Pipes together the cli arguments with the execution +#[derive(Parser)] +pub struct SimulationApp { + /// Json file path, on `SimulationSettings` format + #[clap(long, short)] + input_settings: PathBuf, + /// Output file path + #[clap(long, short)] + output_file: PathBuf, + /// Output format selector + #[clap(long, short = 'f', default_value_t)] + output_format: OutputFormat, +} + +impl SimulationApp { + pub fn run(self) -> Result<(), Box> { + let Self { + input_settings, + output_file, + output_format, + } = self; + let simulation_settings: SimulationSettings = load_json_from_file(&input_settings)?; + simulation_settings.distribution.check_distribution()?; + simulation_settings + .byzantine_settings + .distribution + .check_distribution()?; + let mut simulation_runner = SimulationRunner::new(simulation_settings); + // build up series vector + let mut out_data: Vec = Vec::new(); + simulation_runner.simulate(Some(&mut out_data)); + let mut dataframe: DataFrame = out_data_to_dataframe(out_data); + dump_dataframe_to(output_format, &mut dataframe, &output_file)?; + Ok(()) + } +} + +fn out_data_to_dataframe(out_data: Vec) -> DataFrame { + let mut cursor = Cursor::new(Vec::new()); + serde_json::to_writer(&mut cursor, &out_data).expect("Dump data to json "); + let dataframe = JsonReader::new(cursor) + .finish() + .expect("Load dataframe from intermediary json"); + + dataframe + .unnest(["state"]) + .expect("Node state should be unnest") +} + +/// Generically load a json file +fn load_json_from_file(path: &Path) -> Result> { + let f = File::open(path).map_err(Box::new)?; + serde_json::from_reader(f).map_err(|e| Box::new(e) as Box) +} + +fn dump_dataframe_to_json(data: &mut DataFrame, out_path: &Path) -> Result<(), Box> { + let out_path = out_path.with_extension("json"); + let f = File::create(out_path)?; + let mut writer = polars::prelude::JsonWriter::new(f); + writer + .finish(data) + .map_err(|e| Box::new(e) as Box) +} + +fn dump_dataframe_to_csv(data: &mut DataFrame, out_path: &Path) -> Result<(), Box> { + let out_path = out_path.with_extension("csv"); + let f = File::create(out_path)?; + let mut writer = polars::prelude::CsvWriter::new(f); + writer + .finish(data) + .map_err(|e| Box::new(e) as Box) +} + +fn dump_dataframe_to_parquet(data: &mut DataFrame, out_path: &Path) -> Result<(), Box> { + let out_path = out_path.with_extension("parquet"); + let f = File::create(out_path)?; + let writer = polars::prelude::ParquetWriter::new(f); + writer + .finish(data) + .map_err(|e| Box::new(e) as Box) +} + +fn dump_dataframe_to( + output_format: OutputFormat, + data: &mut DataFrame, + out_path: &Path, +) -> Result<(), Box> { + match output_format { + OutputFormat::Json => dump_dataframe_to_json(data, out_path), + OutputFormat::Csv => dump_dataframe_to_csv(data, out_path), + OutputFormat::Parquet => dump_dataframe_to_parquet(data, out_path), + } +} diff --git a/simulations/snow-family/src/main.rs b/simulations/snow-family/src/main.rs new file mode 100644 index 0000000..d0ae0ec --- /dev/null +++ b/simulations/snow-family/src/main.rs @@ -0,0 +1,17 @@ +mod app; +mod network_behaviour; +mod node; +mod output_processors; +mod runner; +mod settings; +mod warding; + +use crate::app::SimulationApp; +use clap::Parser; +use std::error::Error; + +fn main() -> Result<(), Box> { + let app: SimulationApp = app::SimulationApp::parse(); + app.run()?; + Ok(()) +} diff --git a/simulations/snow-family/src/network_behaviour/drop.rs b/simulations/snow-family/src/network_behaviour/drop.rs new file mode 100644 index 0000000..e15c156 --- /dev/null +++ b/simulations/snow-family/src/network_behaviour/drop.rs @@ -0,0 +1,61 @@ +use crate::network_behaviour::NetworkBehaviour; +use crate::node::Vote; +use rand::prelude::IteratorRandom; +use rand::rngs::SmallRng; +use serde::Deserialize; + +/// Randomly drop some of the network votes +/// Drop rate should be normalized +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct RandomDrop { + drop_rate: f32, +} + +impl NetworkBehaviour for RandomDrop { + fn modify_network_state(&mut self, network_state: &mut [Option], rng: &mut SmallRng) { + let amount: usize = + (self.drop_rate.clamp(0f32, 1f32) * network_state.len() as f32).round() as usize; + for i in (0..network_state.len()).choose_multiple(rng, amount) { + *network_state.get_mut(i).unwrap() = None; + } + } +} + +#[cfg(test)] +mod test { + use crate::network_behaviour::drop::RandomDrop; + use crate::network_behaviour::NetworkBehaviour; + use crate::node::{NoTx, Vote}; + use rand::prelude::SmallRng; + use rand::SeedableRng; + + const SEED: u64 = 18042022; + + #[test] + fn full_drop_rate() { + let mut rng: SmallRng = SmallRng::seed_from_u64(SEED); + let mut random_drop = RandomDrop { drop_rate: 1.0 }; + let mut votes: Vec> = (0..10).map(|_| Some(Vote::Yes(NoTx))).collect(); + random_drop.modify_network_state(&mut votes, &mut rng); + assert!(votes.iter().all(Option::is_none)); + } + + #[test] + fn none_drop_rate() { + let mut rng: SmallRng = SmallRng::seed_from_u64(SEED); + let mut random_drop = RandomDrop { drop_rate: 0.0 }; + let mut votes: Vec> = (0..10).map(|_| Some(Vote::Yes(NoTx))).collect(); + random_drop.modify_network_state(&mut votes, &mut rng); + assert!(votes.iter().all(Option::is_some)); + } + + #[test] + fn half_drop_rate() { + let mut rng: SmallRng = SmallRng::seed_from_u64(SEED); + let mut random_drop = RandomDrop { drop_rate: 0.5 }; + let mut votes: Vec> = (0..10).map(|_| Some(Vote::Yes(NoTx))).collect(); + random_drop.modify_network_state(&mut votes, &mut rng); + assert_eq!(votes.iter().filter(|vote| vote.is_some()).count(), 5); + } +} diff --git a/simulations/snow-family/src/network_behaviour/mod.rs b/simulations/snow-family/src/network_behaviour/mod.rs new file mode 100644 index 0000000..65a3298 --- /dev/null +++ b/simulations/snow-family/src/network_behaviour/mod.rs @@ -0,0 +1,34 @@ +mod drop; + +use crate::node::Vote; +use rand::rngs::SmallRng; +use serde::Deserialize; + +/// Modify a ['crate::node::NetworkState'](network state), single exclusive access is guaranteed +pub trait NetworkBehaviour { + fn modify_network_state(&mut self, network_state: &mut [Option], rng: &mut SmallRng); +} + +/// [`NetworkBehaviour`] dispatcher +/// Enum to avoid Boxing (Box) modifiers. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NetworkModifiers { + RandomDrop(drop::RandomDrop), +} + +impl NetworkModifiers { + /// Get inner [`NetworkBehaviour`] mut reference + pub fn network_behaviour_mut(&mut self) -> &mut dyn NetworkBehaviour { + match self { + NetworkModifiers::RandomDrop(behaviour) => behaviour, + } + } +} + +impl NetworkBehaviour for NetworkModifiers { + fn modify_network_state(&mut self, network_state: &mut [Option], rng: &mut SmallRng) { + self.network_behaviour_mut() + .modify_network_state(network_state, rng); + } +} diff --git a/simulations/snow-family/src/node/claro.rs b/simulations/snow-family/src/node/claro.rs new file mode 100644 index 0000000..be68171 --- /dev/null +++ b/simulations/snow-family/src/node/claro.rs @@ -0,0 +1,78 @@ +// std +// crates +use rand::rngs::SmallRng; +use serde::Serialize; +// internal +use crate::node::{query_network_state, ComputeNode, Decision, NetworkState, NoTx, NodeId}; +use crate::output_processors::{NodeStateRecord, SerializedNodeState}; +use claro::{ClaroSolver, ClaroState}; + +/// Claro consensus node +/// Wrapper over [`::claro::ClaroSolver`] +pub struct ClaroNode { + solver: ClaroSolver, + network_state: NetworkState, + node_id: NodeId, + rng: SmallRng, +} + +impl ClaroNode { + pub fn new( + node_id: usize, + solver: ClaroSolver, + network_state: NetworkState, + rng: SmallRng, + ) -> Self { + Self { + node_id, + solver, + network_state, + rng, + } + } +} + +impl ComputeNode for ClaroNode { + fn id(&self) -> usize { + self.node_id + } + + fn step(&mut self) { + if matches!(self.solver.decision(), Decision::Undecided(_)) { + let votes = query_network_state( + &self.network_state, + self.solver.node_query().query_size(), + self.node_id, + &mut self.rng, + ); + self.solver.step(NoTx, &votes); + } + } + + fn decision(&self) -> Decision { + self.solver.decision() + } +} + +#[derive(Serialize)] +struct OutClaroState { + evidence: u64, + evidence_accumulated: u64, + confidence: u64, +} + +impl From<&ClaroState> for OutClaroState { + fn from(state: &ClaroState) -> Self { + OutClaroState { + evidence: state.evidence() as u64, + evidence_accumulated: state.evidence_accumulated() as u64, + confidence: state.confidence() as u64, + } + } +} + +impl NodeStateRecord for ClaroNode { + fn get_serialized_state_record(&self) -> SerializedNodeState { + serde_json::to_value(OutClaroState::from(self.solver.state())).unwrap() + } +} diff --git a/simulations/snow-family/src/node/infantile.rs b/simulations/snow-family/src/node/infantile.rs new file mode 100644 index 0000000..1879fb5 --- /dev/null +++ b/simulations/snow-family/src/node/infantile.rs @@ -0,0 +1,73 @@ +use rand::rngs::SmallRng; +// std +// crates +// internal +use crate::node::{ + query_network_state, ComputeNode, Decision, NetworkState, NoTx, NodeId, Opinion, Vote, +}; +use crate::output_processors::NodeStateRecord; + +/// Node that replies with the opposite of the step query. +/// For each query: +/// if majority == yes: reply no +/// if majority == no: reply yes +pub struct InfantileNode { + network_state: NetworkState, + query_size: usize, + node_id: NodeId, + decision: Decision, + rng: SmallRng, +} + +impl InfantileNode { + pub fn new( + node_id: usize, + query_size: usize, + network_state: NetworkState, + rng: SmallRng, + ) -> Self { + let decision = Decision::Undecided(Opinion::None(NoTx)); + Self { + node_id, + query_size, + network_state, + decision, + rng, + } + } + + fn flip_majority(votes: &[Vote]) -> Opinion { + let yes_votes = votes + .iter() + .filter(|vote| matches!(vote, Vote::Yes(_))) + .count(); + let len = votes.len(); + if yes_votes > len / 2 { + Opinion::No(NoTx) + } else { + Opinion::Yes(NoTx) + } + } +} + +impl ComputeNode for InfantileNode { + fn id(&self) -> usize { + self.node_id + } + + fn step(&mut self) { + let votes = query_network_state( + &self.network_state, + self.query_size, + self.node_id, + &mut self.rng, + ); + self.decision = Decision::Undecided(InfantileNode::flip_majority(&votes)); + } + + fn decision(&self) -> Decision { + self.decision + } +} + +impl NodeStateRecord for InfantileNode {} diff --git a/simulations/snow-family/src/node/mod.rs b/simulations/snow-family/src/node/mod.rs new file mode 100644 index 0000000..04fd338 --- /dev/null +++ b/simulations/snow-family/src/node/mod.rs @@ -0,0 +1,184 @@ +// std +use std::sync::{Arc, RwLock}; +// crates +use ::claro::ClaroSolver; +use rand::prelude::IteratorRandom; +use rand::rngs::SmallRng; +use rand::RngCore; +// internal +use crate::node::claro::ClaroNode; +use crate::node::infantile::InfantileNode; +pub use crate::node::omniscient::{MasterOmniscientNode, OmniscientPuppetNode}; +use crate::node::random::RandomNode; +use crate::node::snowball::SnowballNode; +use crate::output_processors::NodeStateRecord; +use ::snowball::SnowballSolver; + +mod claro; +mod infantile; +mod omniscient; +mod random; +mod snowball; + +/// Consensus experiments consist on just one round, we just care about voting itself not the content +/// hence we need a Transaction that carries no information. +#[derive(Copy, Clone, Debug)] +pub struct NoTx; + +/// NoTx vote +pub type Vote = ::claro::Vote; + +/// NoTx decision +pub type Decision = ::claro::Decision; + +/// NoTx opinion +pub type Opinion = ::claro::Opinion; + +pub type NodeId = usize; + +/// Shared hook to the simulation state +pub type NetworkState = Arc>>>; + +/// Node computation abstraction layer +pub trait ComputeNode { + fn id(&self) -> usize; + + fn step(&mut self); + + fn vote(&self) -> Option { + self.opinion().into() + } + + fn opinion(&self) -> Opinion { + match self.decision() { + Decision::Decided(opinion) | Decision::Undecided(opinion) => opinion, + } + } + + fn decision(&self) -> Decision; +} + +/// Query the network state for a fixed size skipping self node id +pub fn query_network_state( + network_state: &NetworkState, + query_size: usize, + node_id: NodeId, + rng: &mut impl RngCore, +) -> Vec { + network_state + .read() + .unwrap() + .iter() + .enumerate() + .choose_multiple(rng, query_size + 1) + .into_iter() + .filter_map(|(id, vote)| if id != node_id { *vote } else { None }) + .take(query_size) + .collect() +} + +/// Node dispatcher +/// Enum to avoid Boxing (Box) the nodes. +pub enum Node { + Snowball(snowball::SnowballNode), + Claro(claro::ClaroNode), + Random(random::RandomNode), + Infantile(infantile::InfantileNode), + OmniscientPuppet(omniscient::OmniscientPuppetNode), +} + +impl Node { + pub fn new_snowball( + node_id: NodeId, + solver: SnowballSolver, + network_state: NetworkState, + rng: SmallRng, + ) -> Self { + Self::Snowball(SnowballNode::new(node_id, solver, network_state, rng)) + } + + pub fn new_claro( + node_id: NodeId, + solver: ClaroSolver, + network_state: NetworkState, + seed: SmallRng, + ) -> Self { + Self::Claro(ClaroNode::new(node_id, solver, network_state, seed)) + } + + pub fn new_random(node_id: NodeId) -> Self { + Self::Random(RandomNode::new(node_id)) + } + + pub fn new_infantile( + node_id: NodeId, + query_size: usize, + network_state: NetworkState, + rng: SmallRng, + ) -> Self { + Self::Infantile(InfantileNode::new(node_id, query_size, network_state, rng)) + } + + pub fn new_omniscient_puppet(puppet: OmniscientPuppetNode) -> Self { + Self::OmniscientPuppet(puppet) + } + + /// Get `ComputeNode` inner mut reference + pub fn inner_node_mut(&mut self) -> &mut dyn ComputeNode { + let node: &mut dyn ComputeNode = match self { + Node::Snowball(node) => node, + Node::Claro(node) => node, + Node::Random(node) => node, + Node::Infantile(node) => node, + Node::OmniscientPuppet(node) => node, + }; + node + } + + /// Get `ComputeNode` inner reference + pub fn inner_node(&self) -> &dyn ComputeNode { + let node: &dyn ComputeNode = match self { + Node::Snowball(node) => node, + Node::Claro(node) => node, + Node::Random(node) => node, + Node::Infantile(node) => node, + Node::OmniscientPuppet(node) => node, + }; + node + } + + pub fn serialized_state(&self) -> &dyn NodeStateRecord { + match self { + Node::Snowball(node) => node, + Node::Claro(node) => node, + Node::Random(node) => node, + Node::Infantile(node) => node, + Node::OmniscientPuppet(node) => node, + } + } + + pub fn type_as_string(&self) -> String { + match self { + Node::Snowball(_) => "snowball", + Node::Claro(_) => "claro", + Node::Random(_) => "random", + Node::Infantile(_) => "infantile", + Node::OmniscientPuppet(_) => "omniscient", + } + .to_string() + } +} + +impl ComputeNode for Node { + fn id(&self) -> usize { + self.inner_node().id() + } + + fn step(&mut self) { + self.inner_node_mut().step() + } + + fn decision(&self) -> Decision { + self.inner_node().decision() + } +} diff --git a/simulations/snow-family/src/node/omniscient.rs b/simulations/snow-family/src/node/omniscient.rs new file mode 100644 index 0000000..be95b40 --- /dev/null +++ b/simulations/snow-family/src/node/omniscient.rs @@ -0,0 +1,114 @@ +// std +use std::sync::{Arc, RwLock}; +// crates +// internal +use crate::node::{ComputeNode, Decision, NetworkState, NoTx, NodeId, Opinion, Vote}; +use crate::output_processors::NodeStateRecord; + +/// Node that knows the network state all the time. +/// It orchestrates responses based on that. +/// As an optimization just a single node takes care of everything, then we place Puppet nodes +/// in the list of nodes that just replies with whatever the Master omniscient node decides. +pub struct MasterOmniscientNode { + honest_nodes_ids: Vec, + omniscient_nodes_ids: Vec, + network_state: NetworkState, + decision: Arc>, + node_id: NodeId, +} + +/// Omniscient puppet node. Node that just replies with whatever the `MasterOmniscientNode` decides. +#[derive(Clone)] +pub struct OmniscientPuppetNode { + node_id: NodeId, + decision: Arc>, +} + +impl MasterOmniscientNode { + pub fn new( + node_id: NodeId, + honest_nodes_ids: Vec, + omniscient_nodes_ids: Vec, + network_state: NetworkState, + ) -> Self { + Self { + node_id, + honest_nodes_ids, + omniscient_nodes_ids, + network_state, + decision: Arc::new(RwLock::new(Decision::Undecided(Opinion::None(NoTx)))), + } + } + + fn analyze_and_write_votes(&mut self) { + let mut state = self + .network_state + .write() + .expect("Only access to network state resource from omniscient node"); + + let honest_votes: Vec> = self + .honest_nodes_ids + .iter() + .map(|node_id| state.get(*node_id).expect("Node id should be within range")) + .copied() + .collect(); + + let yes_votes = honest_votes + .iter() + .filter(|v| matches!(v, Some(Vote::Yes(_)))) + .count(); + let no_votes = honest_votes + .iter() + .filter(|v| matches!(v, Some(Vote::No(_)))) + .count(); + + let vote = if yes_votes > no_votes { + *self.decision.write().unwrap() = Decision::Undecided(Opinion::No(NoTx)); + Some(Vote::No(NoTx)) + } else { + *self.decision.write().unwrap() = Decision::Undecided(Opinion::Yes(NoTx)); + Some(Vote::Yes(NoTx)) + }; + + for &i in &self.omniscient_nodes_ids { + if let Some(old_vote) = state.get_mut(i) { + *old_vote = vote; + } + } + } + + pub fn puppet_node(&self, node_id: NodeId) -> OmniscientPuppetNode { + OmniscientPuppetNode { + node_id, + decision: Arc::clone(&self.decision), + } + } +} + +impl ComputeNode for MasterOmniscientNode { + fn id(&self) -> usize { + self.node_id + } + + fn step(&mut self) { + self.analyze_and_write_votes(); + } + + fn decision(&self) -> Decision { + *self.decision.read().unwrap() + } +} + +impl ComputeNode for OmniscientPuppetNode { + fn id(&self) -> usize { + self.node_id + } + + fn step(&mut self) {} + + fn decision(&self) -> Decision { + *self.decision.read().unwrap() + } +} + +impl NodeStateRecord for OmniscientPuppetNode {} diff --git a/simulations/snow-family/src/node/random.rs b/simulations/snow-family/src/node/random.rs new file mode 100644 index 0000000..18a80f8 --- /dev/null +++ b/simulations/snow-family/src/node/random.rs @@ -0,0 +1,45 @@ +// std +// crates +// internal +use crate::node::{ComputeNode, Decision, NoTx, NodeId, Opinion}; +use crate::output_processors::NodeStateRecord; + +/// Nodes that takes a random decision each step +pub struct RandomNode { + decision: Decision, + node_id: NodeId, +} + +impl RandomNode { + pub fn new(node_id: NodeId) -> Self { + Self { + decision: Decision::Undecided(Opinion::None(NoTx)), + node_id, + } + } + + fn rand_opinion() -> Opinion { + let bool_opinion: bool = rand::random(); + if bool_opinion { + Opinion::Yes(NoTx) + } else { + Opinion::No(NoTx) + } + } +} + +impl ComputeNode for RandomNode { + fn id(&self) -> usize { + self.node_id + } + + fn step(&mut self) { + self.decision = Decision::Undecided(RandomNode::rand_opinion()); + } + + fn decision(&self) -> Decision { + self.decision + } +} + +impl NodeStateRecord for RandomNode {} diff --git a/simulations/snow-family/src/node/snowball.rs b/simulations/snow-family/src/node/snowball.rs new file mode 100644 index 0000000..0932dc7 --- /dev/null +++ b/simulations/snow-family/src/node/snowball.rs @@ -0,0 +1,70 @@ +use rand::rngs::SmallRng; +// std +// crates +use serde::Serialize; +// internal +use crate::node::{query_network_state, ComputeNode, Decision, NetworkState, NoTx, NodeId}; +use crate::output_processors::{NodeStateRecord, SerializedNodeState}; +use snowball::SnowballSolver; + +/// Snowball consensus node +/// Wrapper over [`::snowball::SnowballSolver`] +pub struct SnowballNode { + solver: SnowballSolver, + network_state: NetworkState, + node_id: NodeId, + rng: SmallRng, +} + +impl SnowballNode { + pub fn new( + node_id: usize, + solver: SnowballSolver, + network_state: NetworkState, + rng: SmallRng, + ) -> Self { + Self { + node_id, + solver, + network_state, + rng, + } + } +} + +impl ComputeNode for SnowballNode { + fn id(&self) -> usize { + self.node_id + } + + fn step(&mut self) { + if matches!(self.solver.decision(), Decision::Undecided(_)) { + let votes = query_network_state( + &self.network_state, + self.solver.node_query().query_size(), + self.node_id, + &mut self.rng, + ); + self.solver.step(&votes); + } + } + + fn decision(&self) -> Decision { + self.solver.decision() + } +} + +#[derive(Serialize)] +struct OutSnowballState { + consecutive_success: u64, +} + +impl NodeStateRecord for SnowballNode { + fn get_serialized_state_record(&self) -> SerializedNodeState { + let consecutive_success = self.solver.consecutive_success(); + serde_json::to_value(OutSnowballState { + consecutive_success, + }) + .unwrap() + } +} diff --git a/simulations/snow-family/src/output_processors/mod.rs b/simulations/snow-family/src/output_processors/mod.rs new file mode 100644 index 0000000..345abb7 --- /dev/null +++ b/simulations/snow-family/src/output_processors/mod.rs @@ -0,0 +1,19 @@ +use serde::Serialize; + +pub type SerializedNodeState = serde_json::Value; + +#[derive(Serialize)] +pub struct OutData { + pub id: u64, + pub iteration: u64, + pub round: u64, + pub vote: u8, + pub _type: String, + pub state: SerializedNodeState, +} + +pub trait NodeStateRecord { + fn get_serialized_state_record(&self) -> SerializedNodeState { + SerializedNodeState::Null + } +} diff --git a/simulations/snow-family/src/runner/async_runner.rs b/simulations/snow-family/src/runner/async_runner.rs new file mode 100644 index 0000000..711455a --- /dev/null +++ b/simulations/snow-family/src/runner/async_runner.rs @@ -0,0 +1,64 @@ +use crate::node::{ComputeNode, Vote}; +use crate::output_processors::OutData; +use crate::runner::SimulationRunner; +use crate::warding::SimulationState; +use rand::prelude::SliceRandom; +use rayon::prelude::*; +use std::collections::HashSet; +use std::sync::Arc; + +pub fn simulate( + runner: &mut SimulationRunner, + chunk_size: usize, + mut out_data: Option<&mut Vec>, +) { + let mut node_ids: Vec = (0..runner + .nodes + .read() + .expect("Read access to nodes vector") + .len()) + .collect(); + let mut simulation_state = SimulationState { + network_state: Arc::clone(&runner.network_state), + nodes: Arc::clone(&runner.nodes), + iteration: 0, + round: 0, + }; + + runner.dump_state_to_out_data(&simulation_state, &mut out_data); + + loop { + node_ids.shuffle(&mut runner.rng); + for ids_chunk in node_ids.chunks(chunk_size) { + if let Some(master_omniscient) = runner.master_omniscient.as_mut() { + master_omniscient.step(); + } + let ids: HashSet = ids_chunk.iter().copied().collect(); + let new_state: Vec> = runner + .nodes + .write() + .expect("Write access to nodes vector") + .par_iter_mut() + .enumerate() + .map(|(id, node)| { + if ids.contains(&id) { + node.step(); + node.vote() + } else { + node.vote() + } + }) + .collect(); + runner.set_new_network_state(new_state); + runner.dump_state_to_out_data(&simulation_state, &mut out_data); + simulation_state.iteration += 1; + } + simulation_state.round += 1; + // check if any condition makes the simulation stop + if runner.check_wards(&simulation_state) { + break; + } + // run modifiers over the current step network state + runner.run_network_behaviour_modifiers(); + } +} diff --git a/simulations/snow-family/src/runner/glauber_runner.rs b/simulations/snow-family/src/runner/glauber_runner.rs new file mode 100644 index 0000000..3db6520 --- /dev/null +++ b/simulations/snow-family/src/runner/glauber_runner.rs @@ -0,0 +1,66 @@ +use crate::node::{ComputeNode, Node, NodeId}; +use crate::output_processors::OutData; +use crate::runner::SimulationRunner; +use crate::warding::SimulationState; +use rand::prelude::IteratorRandom; +use std::collections::BTreeSet; +use std::sync::Arc; + +/// [Glauber dynamics simulation](https://en.wikipedia.org/wiki/Glauber_dynamics) +pub fn simulate( + runner: &mut SimulationRunner, + update_rate: usize, + maximum_iterations: usize, + mut out_data: Option<&mut Vec>, +) { + let mut simulation_state = SimulationState { + network_state: Arc::clone(&runner.network_state), + nodes: Arc::clone(&runner.nodes), + iteration: 0, + round: 0, + }; + let mut nodes_remaining: BTreeSet = (0..runner + .nodes + .read() + .expect("Read access to nodes vector") + .len()) + .collect(); + let iterations: Vec<_> = (0..maximum_iterations).collect(); + 'main: for chunk in iterations.chunks(update_rate) { + for i in chunk { + simulation_state.iteration = *i; + if nodes_remaining.is_empty() { + break 'main; + } + + let node_id = *nodes_remaining.iter().choose(&mut runner.rng).expect( + "Some id to be selected as it should be impossible for the set to be empty here", + ); + + { + let vote = { + let mut shared_nodes = + runner.nodes.write().expect("Write access to nodes vector"); + let node: &mut Node = shared_nodes + .get_mut(node_id) + .expect("Node should be present"); + + node.step(); + if matches!(node.decision(), claro::Decision::Decided(_)) { + nodes_remaining.remove(&node_id); + } + node.vote() + }; + runner.update_single_network_state_vote(node_id, vote); + } + + // check if any condition makes the simulation stop + if runner.check_wards(&simulation_state) { + break 'main; + } + // run modifiers over the current step network state + runner.run_network_behaviour_modifiers(); + } + runner.dump_state_to_out_data(&simulation_state, &mut out_data); + } +} diff --git a/simulations/snow-family/src/runner/mod.rs b/simulations/snow-family/src/runner/mod.rs new file mode 100644 index 0000000..0ae86ac --- /dev/null +++ b/simulations/snow-family/src/runner/mod.rs @@ -0,0 +1,448 @@ +mod async_runner; +mod glauber_runner; +mod sync_runner; + +// std +use std::sync::{Arc, RwLock}; +// crates +use rand::prelude::SliceRandom; +use rand::rngs::SmallRng; +use rand::{RngCore, SeedableRng}; +use rayon::prelude::*; +// internal +use crate::network_behaviour::NetworkBehaviour; +use crate::node::{ + ComputeNode, MasterOmniscientNode, NetworkState, NoTx, Node, NodeId, Opinion, Vote, +}; +use crate::output_processors::OutData; +use crate::settings::{ + ByzantineDistribution, ByzantineSettings, ConsensusSettings, SimulationSettings, + SimulationStyle, +}; +use crate::warding::{SimulationState, SimulationWard}; +use claro::{ClaroSolver, NodeQuery}; +use snowball::SnowballSolver; + +/// Encapsulation solution for the simulations runner +/// Holds the network state, the simulating nodes and the simulation settings. +pub struct SimulationRunner { + network_state: NetworkState, + nodes: Arc>>, + master_omniscient: Option, + settings: SimulationSettings, + rng: SmallRng, +} + +impl SimulationRunner { + pub fn new(settings: SimulationSettings) -> Self { + let seed = settings + .seed + .unwrap_or_else(|| rand::thread_rng().next_u64()); + + println!("Seed: {}", seed); + + let mut rng = SmallRng::seed_from_u64(seed); + + let (nodes, network_state, master_omniscient) = + Self::nodes_from_initial_settings(&settings, &mut rng); + + let nodes = Arc::new(RwLock::new(nodes)); + + Self { + network_state, + nodes, + master_omniscient, + settings, + rng, + } + } + + /// Initialize nodes from settings and calculate initial network state. + fn nodes_from_initial_settings( + settings: &SimulationSettings, + mut seed: &mut SmallRng, + ) -> (Vec, NetworkState, Option) { + let SimulationSettings { + consensus_settings, + distribution, + byzantine_settings: + ByzantineSettings { + total_size, + distribution: + ByzantineDistribution { + honest, + infantile, + random, + omniscient, + }, + }, + .. + } = settings; + + // shuffling is just for representation + let mut node_ids: Vec<_> = (0..*total_size).collect(); + node_ids.shuffle(seed); + let mut node_ids_iter = node_ids.into_iter(); + + // total sized based sizes + let [honest_size, infantile_size, random_size, omniscient_size] = + [honest, infantile, random, omniscient] + .map(|&x| (*total_size as f32 * x).round() as usize); + + dbg!([honest_size, infantile_size, random_size, omniscient_size]); + + let options = [Opinion::None(NoTx), Opinion::Yes(NoTx), Opinion::No(NoTx)]; + + // build up initial hones nodes distribution + let mut votes_distribution: Vec = options + .into_iter() + .flat_map(|opinion| { + let size: usize = + (honest_size as f32 * distribution.weight_by_opinion(&opinion)) as usize; + std::iter::repeat(opinion).take(size) + }) + .chain(std::iter::repeat(Opinion::None(NoTx))) + .take(honest_size) + .collect(); + + // check that we actually have all opinions as needed + assert_eq!(votes_distribution.len(), honest_size); + + // shuffle distribution + votes_distribution.shuffle(seed); + + // uninitialized network state, should be recalculated afterwards + let network_state: NetworkState = Arc::new(RwLock::new(vec![None; *total_size])); + + // Allow needless collect: we actually need to do so in order to liberate the node_ids_iter + // otherwise it is borrowed mutably more than once...apparently the compiler is not smart enough (still) + // to catch that it should be safe to do so in this case. So, we collect. + // This should not really impact on running performance other than getting the nodes prepared + // would take a bit more. + let hones_nodes_ids: Vec<_> = std::iter::from_fn(|| node_ids_iter.next()) + .take(honest_size) + .collect(); + + #[allow(clippy::needless_collect)] + let honest_nodes: Vec<_> = Self::build_honest_nodes( + hones_nodes_ids.iter().copied().zip(votes_distribution), + *total_size, + Arc::clone(&network_state), + *consensus_settings, + seed, + ) + .collect(); + + #[allow(clippy::needless_collect)] + let infantile_nodes: Vec<_> = std::iter::from_fn(|| node_ids_iter.next()) + .take(infantile_size) + .map(|node_id| { + Node::new_infantile( + node_id, + consensus_settings.query_size(), + Arc::clone(&network_state), + SmallRng::from_rng(&mut seed).expect("Rng should build properly from seed rng"), + ) + }) + .collect(); + + #[allow(clippy::needless_collect)] + let random_nodes: Vec<_> = std::iter::from_fn(|| node_ids_iter.next()) + .take(random_size) + .map(Node::new_random) + .collect(); + + let (master_omniscient, omniscient_nodes) = { + if omniscient_size > 0 { + let omniscient_nodes_ids: Vec<_> = std::iter::from_fn(|| node_ids_iter.next()) + .take(omniscient_size) + .collect(); + + let omniscient_node = MasterOmniscientNode::new( + NodeId::MAX, + hones_nodes_ids, + omniscient_nodes_ids.clone(), + Arc::clone(&network_state), + ); + + #[allow(clippy::needless_collect)] + let puppets: Vec<_> = omniscient_nodes_ids + .iter() + .map(|id| Node::new_omniscient_puppet(omniscient_node.puppet_node(*id))) + .collect(); + + (Some(omniscient_node), puppets.into_iter()) + } else { + (None, vec![].into_iter()) + } + }; + + let mut nodes: Vec = honest_nodes + .into_iter() + .chain(omniscient_nodes) + .chain(infantile_nodes.into_iter()) + .chain(random_nodes.into_iter()) + .collect(); + + nodes.sort_unstable_by_key(|node| node.inner_node().id()); + + // set up network state with the current distribution + let new_network_state = Self::network_state_from_nodes(&nodes); + *network_state.write().unwrap() = new_network_state; + (nodes, network_state, master_omniscient) + } + + fn build_honest_nodes<'a>( + node_data: impl Iterator + 'a, + total_size: usize, + network_state: NetworkState, + consensus_settings: ConsensusSettings, + mut seed: &'a mut SmallRng, + ) -> impl Iterator + 'a { + match consensus_settings { + ConsensusSettings::SnowBall(snowball_settings) => { + node_data.map(Box::new(move |(node_id, opinion)| { + Node::new_snowball( + node_id, + SnowballSolver::with_initial_opinion( + snowball_settings, + NodeQuery::new(total_size, node_id.to_string()), + opinion, + ), + Arc::clone(&network_state), + SmallRng::from_rng(&mut seed) + .expect("Rng should build properly from seed rng"), + ) + }) + as Box Node>) + } + ConsensusSettings::Claro(claro_settings) => { + node_data.map(Box::new(move |(node_id, opinion)| { + Node::new_claro( + node_id, + ClaroSolver::with_initial_opinion( + claro_settings, + NodeQuery::new(total_size, node_id.to_string()), + opinion, + ), + Arc::clone(&network_state), + SmallRng::from_rng(&mut seed) + .expect("Rng should build properly from seed rng"), + ) + }) + as Box Node>) + } + } + } + + #[inline] + fn network_state_from_nodes(nodes: &[Node]) -> Vec> { + dbg!(nodes.len()); + nodes.par_iter().map(|node| node.vote()).collect() + } + + pub fn simulate(&mut self, out_data: Option<&mut Vec>) { + match &self.settings.simulation_style { + SimulationStyle::Sync => { + sync_runner::simulate(self, out_data); + } + &SimulationStyle::Async { chunks } => { + async_runner::simulate(self, chunks, out_data); + } + &SimulationStyle::Glauber { + maximum_iterations, + update_rate, + } => { + glauber_runner::simulate(self, update_rate, maximum_iterations, out_data); + } + } + } + + fn dump_state_to_out_data( + &self, + simulation_state: &SimulationState, + out_ata: &mut Option<&mut Vec>, + ) { + if let Some(out) = out_ata.as_deref_mut() { + let nodes = self.nodes.read().unwrap(); + let iteration = simulation_state.iteration as u64; + let round = simulation_state.round as u64; + let updated = nodes.iter().map(|node| { + let node_type = node.type_as_string(); + let vote = match node.vote() { + None => 0u8, + Some(Vote::Yes(_)) => 1, + Some(Vote::No(_)) => 2, + }; + OutData { + id: node.inner_node().id() as u64, + iteration, + _type: node_type, + round, + vote, + state: node.serialized_state().get_serialized_state_record(), + } + }); + + out.extend(updated); + } + } + + fn check_wards(&mut self, state: &SimulationState) -> bool { + self.settings + .wards + .par_iter_mut() + .map(|ward| ward.analyze(state)) + .any(|x| x) + } + + fn run_network_behaviour_modifiers(&mut self) { + let mut network_state = self + .network_state + .write() + .expect("Single access to network state for running behaviour modifiers"); + + for modifier in self.settings.network_modifiers.iter_mut() { + modifier.modify_network_state(&mut network_state, &mut self.rng); + } + } + + pub fn step(&mut self) { + let new_network_state: Vec> = self.run_step(); + self.set_new_network_state(new_network_state); + } + + fn set_new_network_state(&mut self, new_network_state: Vec>) { + let mut network_state = self + .network_state + .write() + .expect("No threads could be accessing the network state"); + + *network_state = new_network_state; + } + + fn update_single_network_state_vote(&mut self, id: NodeId, vote: Option) { + let mut network_state = self + .network_state + .write() + .expect("No threads could be accessing the network state"); + + *network_state.get_mut(id).unwrap() = vote; + } + + fn run_step(&mut self) -> Vec> { + if let Some(master_omniscient) = self.master_omniscient.as_mut() { + master_omniscient.step(); + } + self.nodes + .write() + .expect("Single access to nodes vector") + .par_iter_mut() + .map(|node| { + node.step(); + node.vote() + }) + .collect() + } +} + +#[cfg(test)] +mod test { + use crate::node::{ComputeNode, Node, Vote}; + use crate::runner::SimulationRunner; + use crate::settings::{ + ByzantineDistribution, ByzantineSettings, ConsensusSettings, InitialDistribution, + SimulationSettings, + }; + use claro::{ClaroConfiguration, QueryConfiguration}; + use rand::rngs::SmallRng; + use rand::{thread_rng, SeedableRng}; + + #[test] + fn nodes_distribution_from_initial_settings() { + let initial_settings = SimulationSettings { + simulation_style: Default::default(), + consensus_settings: ConsensusSettings::Claro(ClaroConfiguration { + evidence_alpha: 0.0, + evidence_alpha_2: 0.0, + confidence_beta: 0.0, + look_ahead: 0, + query: QueryConfiguration { + query_size: 0, + initial_query_size: 0, + query_multiplier: 0, + max_multiplier: 0, + }, + }), + distribution: InitialDistribution { + yes: 0.5, + no: 0.5, + none: 0.0, + }, + byzantine_settings: ByzantineSettings { + total_size: 999, + distribution: ByzantineDistribution { + honest: 0.7, + infantile: 0.1, + random: 0.1, + omniscient: 0.1, + }, + }, + wards: vec![], + network_modifiers: vec![], + seed: None, + }; + let mut rng = SmallRng::from_rng(&mut thread_rng()).unwrap(); + let (nodes, _, _) = + SimulationRunner::nodes_from_initial_settings(&initial_settings, &mut rng); + let honest_nodes: Vec<_> = nodes + .iter() + .filter(|node| matches!(node, Node::Claro(_))) + .collect(); + + assert_eq!( + honest_nodes.len(), + (initial_settings.byzantine_settings.total_size as f32 + * initial_settings.byzantine_settings.distribution.honest) as usize + ); + + let half_count = honest_nodes.len() / 2; + + let yes_count = honest_nodes + .iter() + .filter(|node| matches!(node.vote(), Some(Vote::Yes(_)))) + .count(); + + assert_eq!(yes_count, half_count); + + let no_count = honest_nodes + .iter() + .filter(|node| matches!(node.vote(), Some(Vote::No(_)))) + .count(); + + assert_eq!(no_count, half_count); + + let byzantine_rate_size = 100; + + let infantile_nodes_count = nodes + .iter() + .filter(|node| matches!(node, Node::Infantile(_))) + .count(); + + assert_eq!(infantile_nodes_count, byzantine_rate_size); + + let random_nodes_count = nodes + .iter() + .filter(|node| matches!(node, Node::Random(_))) + .count(); + + assert_eq!(random_nodes_count, byzantine_rate_size); + + let omniscient_nodes_count = nodes + .iter() + .filter(|node| matches!(node, Node::OmniscientPuppet(_))) + .count(); + + assert_eq!(omniscient_nodes_count, byzantine_rate_size); + } +} diff --git a/simulations/snow-family/src/runner/sync_runner.rs b/simulations/snow-family/src/runner/sync_runner.rs new file mode 100644 index 0000000..60aa4c0 --- /dev/null +++ b/simulations/snow-family/src/runner/sync_runner.rs @@ -0,0 +1,29 @@ +use super::SimulationRunner; +use crate::output_processors::OutData; +use crate::warding::SimulationState; +use std::sync::Arc; + +/// Simulate with option of dumping the network state as a `::polars::Series` +pub fn simulate(runner: &mut SimulationRunner, mut out_data: Option<&mut Vec>) { + let mut state = SimulationState { + network_state: Arc::clone(&runner.network_state), + nodes: Arc::clone(&runner.nodes), + iteration: 0, + round: 0, + }; + + runner.dump_state_to_out_data(&state, &mut out_data); + + for i in 1.. { + state.round = i; + state.iteration = i; + runner.step(); + runner.dump_state_to_out_data(&state, &mut out_data); + // check if any condition makes the simulation stop + if runner.check_wards(&state) { + break; + } + // run modifiers over the current step network state + runner.run_network_behaviour_modifiers(); + } +} diff --git a/simulations/snow-family/src/settings.rs b/simulations/snow-family/src/settings.rs new file mode 100644 index 0000000..aac9325 --- /dev/null +++ b/simulations/snow-family/src/settings.rs @@ -0,0 +1,155 @@ +use std::error::Error; +use std::fmt::Debug; +// std +// crates +use crate::network_behaviour::NetworkModifiers; +use crate::node::Opinion; +use crate::warding::Ward; +use serde::Deserialize; +// internal + +/// Foreign Serialize, Deserialize implementation for `::snowball::SnowballConfiguration` +#[derive(Debug, Deserialize)] +#[serde(remote = "::snowball::SnowballConfiguration")] +pub struct SnowballConfigurationDeSer { + pub quorum_size: usize, + pub sample_size: usize, + pub decision_threshold: usize, +} + +/// Foreign Serialize, Deserialize implementation for `::claro::QueryConfiguration` +#[derive(Debug, Deserialize)] +#[serde(remote = "::claro::QueryConfiguration")] +pub struct QueryConfigurationDeSer { + pub query_size: usize, + pub initial_query_size: usize, + pub query_multiplier: usize, + pub max_multiplier: usize, +} + +/// Foreign Serialize, Deserialize implementation for `::claro::ClaroConfiguration` +#[derive(Debug, Deserialize)] +#[serde(remote = "::claro::ClaroConfiguration")] +pub struct ClaroConfigurationDeSer { + pub evidence_alpha: f32, + pub evidence_alpha_2: f32, + pub confidence_beta: f32, + pub look_ahead: usize, + #[serde(with = "QueryConfigurationDeSer")] + pub query: ::claro::QueryConfiguration, +} + +/// Consensus selector +#[derive(Debug, Copy, Clone, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ConsensusSettings { + SnowBall(#[serde(with = "SnowballConfigurationDeSer")] ::snowball::SnowballConfiguration), + Claro(#[serde(with = "ClaroConfigurationDeSer")] ::claro::ClaroConfiguration), +} + +impl ConsensusSettings { + pub fn query_size(&self) -> usize { + match self { + ConsensusSettings::SnowBall(snowball) => snowball.sample_size, + ConsensusSettings::Claro(claro) => claro.query.query_size, + } + } +} + +/// Initial normalized distribution settings for hones nodes. Must sum up to `1.0` +#[derive(Debug, Deserialize)] +pub struct InitialDistribution { + pub yes: f32, + pub no: f32, + pub none: f32, +} + +impl InitialDistribution { + pub fn weight_by_opinion(&self, opinion: &Opinion) -> f32 { + match opinion { + Opinion::None(_) => self.none, + Opinion::Yes(_) => self.yes, + Opinion::No(_) => self.no, + } + } + + pub fn check_distribution(&self) -> Result<(), Box> { + let values = [self.none, self.yes, self.no]; + check_normalized_distribution(self, &values) + } +} + +/// Byzantine nodes normalized distribution. Must sum up to `1.0` +#[derive(Debug, Deserialize)] +pub struct ByzantineDistribution { + pub honest: f32, + pub infantile: f32, + pub random: f32, + pub omniscient: f32, +} + +impl ByzantineDistribution { + pub fn check_distribution(&self) -> Result<(), Box> { + let values = [self.honest, self.infantile, self.random, self.omniscient]; + check_normalized_distribution(self, &values) + } +} + +/// Byzantine settings, size of simulation and byzantine distribution +#[derive(Debug, Deserialize)] +pub struct ByzantineSettings { + pub total_size: usize, + pub distribution: ByzantineDistribution, +} + +#[derive(Debug, Deserialize, Default)] +pub enum SimulationStyle { + #[default] + Sync, + Async { + chunks: usize, + }, + Glauber { + maximum_iterations: usize, + update_rate: usize, + }, +} + +/// Full simulation settings: +/// * consensus settings +/// * initial distribution +/// * byzantine setting +/// * simulation wards +/// * simulation network behaviour modifiers +/// * simulation style +#[derive(Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct SimulationSettings { + pub consensus_settings: ConsensusSettings, + pub distribution: InitialDistribution, + pub byzantine_settings: ByzantineSettings, + #[serde(default)] + pub wards: Vec, + #[serde(default)] + pub network_modifiers: Vec, + #[serde(default)] + pub simulation_style: SimulationStyle, + #[serde(default)] + pub seed: Option, +} + +/// Check if a settings distribution is normalized (sum up to `1.0`) +fn check_normalized_distribution( + holder: T, + distribution: &[f32], +) -> Result<(), Box> { + let value: f32 = distribution.iter().sum(); + if value != 1.0f32 { + Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("{holder:?} distribution is not normalized, values sum {value} != 1.0"), + ))) + } else { + Ok(()) + } +} diff --git a/simulations/snow-family/src/warding/converged.rs b/simulations/snow-family/src/warding/converged.rs new file mode 100644 index 0000000..2d06790 --- /dev/null +++ b/simulations/snow-family/src/warding/converged.rs @@ -0,0 +1,87 @@ +use crate::node::{ComputeNode, Decision, Node}; +use crate::warding::{SimulationState, SimulationWard}; +use serde::de::Error; +use serde::{Deserialize, Deserializer}; + +#[derive(Debug, Deserialize)] +pub struct ConvergedWard { + #[serde(deserialize_with = "deserialize_normalized_value")] + ratio: f32, +} + +impl ConvergedWard { + pub fn converged(&self, len: usize, decisions: impl Iterator) -> bool { + let total_decided = decisions + .filter(|decision| matches!(decision, Decision::Decided(_))) + .count(); + + (total_decided as f32 / len as f32) >= self.ratio + } +} + +impl SimulationWard for ConvergedWard { + type SimulationState = SimulationState; + + fn analyze(&mut self, state: &Self::SimulationState) -> bool { + let nodes = state.nodes.read().expect("Read access to nodes vec"); + self.converged(nodes.len(), nodes.iter().map(Node::decision)) + } +} + +// TODO: Probably a good idea to have a serde_utils crate +fn deserialize_normalized_value<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let value = f32::deserialize(deserializer)?; + (0f32..=1f32) + .contains(&value) + .then_some(value) + .ok_or_else(|| { + D::Error::custom(&format!( + "Only normalized values [0.0, 1.0] are valid, got: {}", + value + )) + }) +} + +#[cfg(test)] +mod tests { + use crate::node::NoTx; + use crate::warding::converged::ConvergedWard; + use claro::{Decision, Opinion}; + + #[test] + fn converge_full() { + let decisions = vec![ + Decision::Decided(Opinion::Yes(NoTx)), + Decision::Decided(Opinion::Yes(NoTx)), + ]; + let ward = ConvergedWard { ratio: 1.0 }; + + assert!(ward.converged(2, decisions.into_iter())); + } + + #[test] + fn converge_ratio() { + let decisions = vec![ + Decision::Decided(Opinion::Yes(NoTx)), + Decision::Decided(Opinion::Yes(NoTx)), + Decision::Undecided(Opinion::Yes(NoTx)), + ]; + let ward = ConvergedWard { ratio: 0.5 }; + + assert!(ward.converged(2, decisions.into_iter())); + } + + #[test] + fn not_converge() { + let decisions = vec![ + Decision::Decided(Opinion::Yes(NoTx)), + Decision::Undecided(Opinion::Yes(NoTx)), + ]; + let ward = ConvergedWard { ratio: 1.0 }; + + assert!(!ward.converged(2, decisions.into_iter())); + } +} diff --git a/simulations/snow-family/src/warding/mod.rs b/simulations/snow-family/src/warding/mod.rs new file mode 100644 index 0000000..4e03a0f --- /dev/null +++ b/simulations/snow-family/src/warding/mod.rs @@ -0,0 +1,52 @@ +use crate::node::{NetworkState, Node}; +use serde::Deserialize; +use std::sync::{Arc, RwLock}; + +mod converged; +mod stabilised; +mod ttf; + +pub struct SimulationState { + pub network_state: NetworkState, + pub nodes: Arc>>, + pub iteration: usize, + pub round: usize, +} + +/// A ward is a computation over the `NetworkState`, it must return true if the state satisfies +/// the warding conditions. It is used to stop the consensus simulation if such condition is reached. +pub trait SimulationWard { + type SimulationState; + fn analyze(&mut self, state: &Self::SimulationState) -> bool; +} + +/// Ward dispatcher +/// Enum to avoid Boxing (Box) wards. +#[derive(Debug, Deserialize)] +pub enum Ward { + #[serde(rename = "time_to_finality")] + Ttf(ttf::TimeToFinalityWard), + #[serde(rename = "stabilised")] + Stabilised(stabilised::StabilisedWard), + #[serde(rename = "converged")] + Converged(converged::ConvergedWard), +} + +impl Ward { + pub fn simulation_ward_mut( + &mut self, + ) -> &mut dyn SimulationWard { + match self { + Ward::Ttf(ward) => ward, + Ward::Stabilised(stabilised) => stabilised, + Ward::Converged(converged) => converged, + } + } +} + +impl SimulationWard for Ward { + type SimulationState = SimulationState; + fn analyze(&mut self, state: &Self::SimulationState) -> bool { + self.simulation_ward_mut().analyze(state) + } +} diff --git a/simulations/snow-family/src/warding/stabilised.rs b/simulations/snow-family/src/warding/stabilised.rs new file mode 100644 index 0000000..cfcbc1e --- /dev/null +++ b/simulations/snow-family/src/warding/stabilised.rs @@ -0,0 +1,148 @@ +// std +use std::collections::HashSet; +// crates +use fixed_slice_deque::FixedSliceDeque; +use serde::{Deserialize, Deserializer}; +// internal +use crate::node::{NetworkState, Vote}; +use crate::warding::{SimulationState, SimulationWard}; + +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum StabilisedCheck { + Iterations { + chunk: usize, + }, + Rounds { + #[serde(default, skip_deserializing)] + last_round: usize, + }, +} + +impl StabilisedCheck { + pub fn should_check(&mut self, state: &SimulationState) -> bool { + match self { + StabilisedCheck::Iterations { chunk } => (state.iteration % *chunk) == 0, + StabilisedCheck::Rounds { last_round } => { + let different_round = *last_round < state.round; + *last_round = state.round; + different_round + } + } + } +} + +#[derive(Debug, Deserialize)] +pub struct StabilisedWard { + #[serde(deserialize_with = "deserialize_fixed_slice_from_usize")] + buffer: FixedSliceDeque<(usize, usize)>, + check: StabilisedCheck, +} + +impl StabilisedWard { + fn is_stabilised(&self) -> bool { + if self.buffer.is_full() { + let set: HashSet<_> = self.buffer.iter().copied().collect(); + return set.len() == 1; + } + false + } + + fn count_state(network_state: NetworkState) -> (usize, usize) { + network_state + .read() + .unwrap() + .iter() + .fold((0, 0), |count @ (yes, no), vote| match vote { + None => count, + Some(Vote::Yes(_)) => (yes + 1, no), + Some(Vote::No(_)) => (yes, no + 1), + }) + } +} + +impl SimulationWard for StabilisedWard { + type SimulationState = SimulationState; + + fn analyze(&mut self, state: &Self::SimulationState) -> bool { + if !self.check.should_check(state) { + return false; + } + self.buffer + .push_back(StabilisedWard::count_state(state.network_state.clone())); + self.is_stabilised() + } +} + +fn deserialize_fixed_slice_from_usize<'d, T, D: Deserializer<'d>>( + d: D, +) -> Result, D::Error> { + let value = usize::deserialize(d)?; + Ok(FixedSliceDeque::new(value)) +} + +#[cfg(test)] +mod tests { + use crate::node::{NoTx, Vote}; + use crate::warding::stabilised::{StabilisedCheck, StabilisedWard}; + use crate::warding::{SimulationState, SimulationWard}; + use fixed_slice_deque::FixedSliceDeque; + use std::sync::{Arc, RwLock}; + + #[test] + fn check_rounds() { + let mut ward = StabilisedWard { + buffer: FixedSliceDeque::new(2), + check: StabilisedCheck::Rounds { last_round: 0 }, + }; + + let mut simulation_state = SimulationState { + network_state: Arc::new(RwLock::new(vec![Some(Vote::Yes(NoTx))])), + nodes: Arc::new(RwLock::new(vec![])), + iteration: 0, + round: 0, + }; + + for i in 0..2 { + simulation_state.round = i; + assert!(!ward.analyze(&simulation_state)); + } + + simulation_state.round = 3; + assert!(ward.analyze(&simulation_state)); + } + + #[test] + fn check_iterations() { + let mut ward = StabilisedWard { + buffer: FixedSliceDeque::new(2), + check: StabilisedCheck::Iterations { chunk: 3 }, + }; + + let mut simulation_state = SimulationState { + network_state: Arc::new(RwLock::new(vec![Some(Vote::Yes(NoTx))])), + nodes: Arc::new(RwLock::new(vec![])), + iteration: 0, + round: 0, + }; + + for i in 0..3 { + simulation_state.iteration = i; + assert!(!ward.analyze(&simulation_state)); + } + + simulation_state.iteration = 3; + assert!(ward.analyze(&simulation_state)); + } + + #[test] + fn deserialize() { + let rounds = r#"{ "buffer" : 3, "check" : { "type": "rounds" } }"#; + let iterations = r#"{ "buffer" : 3, "check" : { "type": "iterations", "chunk": 100 } }"#; + for s in [rounds, iterations] { + let ward: StabilisedWard = + serde_json::from_str(s).expect("Should deserialize correctly"); + assert_eq!(ward.buffer.capacity(), 3); + } + } +} diff --git a/simulations/snow-family/src/warding/ttf.rs b/simulations/snow-family/src/warding/ttf.rs new file mode 100644 index 0000000..6c287b4 --- /dev/null +++ b/simulations/snow-family/src/warding/ttf.rs @@ -0,0 +1,42 @@ +use crate::warding::{SimulationState, SimulationWard}; +use serde::Deserialize; + +/// Time to finality ward. It monitors the amount of rounds of the simulations, triggers when surpassing +/// the set threshold. +#[derive(Debug, Deserialize, Copy, Clone)] +pub struct TimeToFinalityWard { + ttf_threshold: usize, +} + +impl SimulationWard for TimeToFinalityWard { + type SimulationState = SimulationState; + fn analyze(&mut self, state: &SimulationState) -> bool { + state.round > self.ttf_threshold + } +} + +#[cfg(test)] +mod test { + use crate::node::NetworkState; + use crate::warding::ttf::TimeToFinalityWard; + use crate::warding::{SimulationState, SimulationWard}; + use std::sync::{Arc, RwLock}; + + #[test] + fn rebase_threshold() { + let network_state = NetworkState::new(RwLock::new(vec![])); + let mut ttf = TimeToFinalityWard { ttf_threshold: 10 }; + let mut cond = false; + let mut state = SimulationState { + network_state, + nodes: Arc::new(Default::default()), + iteration: 0, + round: 0, + }; + for _ in 0..11 { + state.round += 1; + cond = ttf.analyze(&state); + } + assert!(cond); + } +}