Humanize array ser/deser (#468)

* humanize array ser/deser

* split fns

* use `const-hex`

* fix fmt

* create `nomos-utils` crate

* Human serde committeeid (#478)

* Human readable serde for CommitteeId

* Deserialize bytes to string if human readable

* Don't allocate if possible in human serde bytes

---------

Co-authored-by: gusto <bacv@users.noreply.github.com>
This commit is contained in:
Al Liu 2023-10-26 23:16:10 +08:00 committed by GitHub
parent 2f9ebbd32f
commit 1553f29bd9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 149 additions and 23 deletions

View File

@ -15,6 +15,7 @@ members = [
"nomos-da/full-replication", "nomos-da/full-replication",
"nomos-http-api", "nomos-http-api",
"nomos-cli", "nomos-cli",
"nomos-utils",
"nodes/nomos-node", "nodes/nomos-node",
"nodes/mixnode", "nodes/mixnode",
"simulations", "simulations",

View File

@ -15,10 +15,11 @@ rand = "0.8"
rand_chacha = "0.3" rand_chacha = "0.3"
thiserror = "1" thiserror = "1"
fraction = { version = "0.13" } fraction = { version = "0.13" }
nomos-utils = { path = "../nomos-utils", optional = true }
[features] [features]
default = [] default = []
serde = ["dep:serde"] serde = ["dep:serde", "nomos-utils/serde"]
simulation = [] simulation = []
[dev-dependencies] [dev-dependencies]

View File

@ -1,8 +1,23 @@
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq, Ord, PartialOrd)] #[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq, Ord, PartialOrd)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(feature = "serde", serde(transparent))]
pub struct BlockId(pub(crate) [u8; 32]); pub struct BlockId(pub(crate) [u8; 32]);
#[cfg(feature = "serde")]
impl serde::Serialize for BlockId {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
nomos_utils::serde::serialize_bytes_array(self.0, serializer)
}
}
#[cfg(feature = "serde")]
impl<'de> serde::de::Deserialize<'de> for BlockId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
nomos_utils::serde::deserialize_bytes_array(deserializer).map(Self)
}
}
impl BlockId { impl BlockId {
pub const fn new(val: [u8; 32]) -> Self { pub const fn new(val: [u8; 32]) -> Self {
Self(val) Self(val)

View File

@ -3,9 +3,25 @@ use std::collections::BTreeSet;
use crate::NodeId; use crate::NodeId;
#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] #[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct CommitteeId(pub(crate) [u8; 32]); pub struct CommitteeId(pub(crate) [u8; 32]);
#[cfg(feature = "serde")]
impl serde::Serialize for CommitteeId {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
nomos_utils::serde::serialize_bytes_array(self.0, serializer)
}
}
#[cfg(feature = "serde")]
impl<'de> serde::de::Deserialize<'de> for CommitteeId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
nomos_utils::serde::deserialize_bytes_array(deserializer).map(Self)
}
}
impl CommitteeId { impl CommitteeId {
pub const fn new(val: [u8; 32]) -> Self { pub const fn new(val: [u8; 32]) -> Self {
Self(val) Self(val)

View File

@ -1,8 +1,23 @@
#[derive(Clone, Copy, Default, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] #[derive(Clone, Copy, Default, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(feature = "serde", serde(transparent))]
pub struct NodeId(pub(crate) [u8; 32]); pub struct NodeId(pub(crate) [u8; 32]);
#[cfg(feature = "serde")]
impl serde::Serialize for NodeId {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
nomos_utils::serde::serialize_bytes_array(self.0, serializer)
}
}
#[cfg(feature = "serde")]
impl<'de> serde::de::Deserialize<'de> for NodeId {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
nomos_utils::serde::deserialize_bytes_array(deserializer).map(Self)
}
}
impl NodeId { impl NodeId {
pub const fn new(val: [u8; 32]) -> Self { pub const fn new(val: [u8; 32]) -> Self {
Self(val) Self(val)

View File

@ -1060,6 +1060,22 @@ pub struct CarnotInfo {
pub committed_blocks: Vec<BlockId>, pub committed_blocks: Vec<BlockId>,
} }
async fn get_mempool_contents<Item, Key>(
mempool: OutboundRelay<MempoolMsg<Item, Key>>,
) -> Result<Box<dyn Iterator<Item = Item> + Send>, tokio::sync::oneshot::error::RecvError> {
let (reply_channel, rx) = tokio::sync::oneshot::channel();
mempool
.send(MempoolMsg::View {
ancestor_hint: BlockId::zeros(),
reply_channel,
})
.await
.unwrap_or_else(|(e, _)| eprintln!("Could not get transactions from mempool {e}"));
rx.await
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use consensus_engine::Block; use consensus_engine::Block;
@ -1098,26 +1114,10 @@ mod tests {
eprintln!("{serialized}"); eprintln!("{serialized}");
assert_eq!( assert_eq!(
serialized, serialized,
r#"{"id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"current_view":1,"highest_voted_view":-1,"local_high_qc":{"view":0,"id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"safe_blocks":[[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],{"view":0,"parent_qc":{"Standard":{"view":0,"id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}},"leader_proof":{"LeaderId":{"leader_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}}}]],"last_view_timeout_qc":null,"committed_blocks":[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]}"# r#"{"id":"0x0000000000000000000000000000000000000000000000000000000000000000","current_view":1,"highest_voted_view":-1,"local_high_qc":{"view":0,"id":"0x0000000000000000000000000000000000000000000000000000000000000000"},"safe_blocks":[["0x0000000000000000000000000000000000000000000000000000000000000000",{"view":0,"parent_qc":{"Standard":{"view":0,"id":"0x0000000000000000000000000000000000000000000000000000000000000000"}},"leader_proof":{"LeaderId":{"leader_id":"0x0000000000000000000000000000000000000000000000000000000000000000"}}}]],"last_view_timeout_qc":null,"committed_blocks":["0x0000000000000000000000000000000000000000000000000000000000000000"]}"#
); );
let deserialized: CarnotInfo = serde_json::from_str(&serialized).unwrap(); let deserialized: CarnotInfo = serde_json::from_str(&serialized).unwrap();
assert_eq!(deserialized, info); assert_eq!(deserialized, info);
} }
} }
async fn get_mempool_contents<Item, Key>(
mempool: OutboundRelay<MempoolMsg<Item, Key>>,
) -> Result<Box<dyn Iterator<Item = Item> + Send>, tokio::sync::oneshot::error::RecvError> {
let (reply_channel, rx) = tokio::sync::oneshot::channel();
mempool
.send(MempoolMsg::View {
ancestor_hint: BlockId::zeros(),
reply_channel,
})
.await
.unwrap_or_else(|(e, _)| eprintln!("Could not get transactions from mempool {e}"));
rx.await
}

11
nomos-utils/Cargo.toml Normal file
View File

@ -0,0 +1,11 @@
[package]
name = "nomos-utils"
version = "0.1.0"
edition = "2021"
[features]
serde = ["dep:serde"]
[dependencies]
const-hex = "1"
serde = { version = "1.0", optional = true }

67
nomos-utils/src/lib.rs Normal file
View File

@ -0,0 +1,67 @@
#[cfg(feature = "serde")]
pub mod serde {
fn serialize_human_readable_bytes_array<const N: usize, S: serde::Serializer>(
src: [u8; N],
serializer: S,
) -> Result<S::Ok, S::Error> {
use serde::Serialize;
const_hex::const_encode::<N, true>(&src)
.as_str()
.serialize(serializer)
}
pub fn serialize_bytes_array<const N: usize, S: serde::Serializer>(
src: [u8; N],
serializer: S,
) -> Result<S::Ok, S::Error> {
if serializer.is_human_readable() {
serialize_human_readable_bytes_array(src, serializer)
} else {
serializer.serialize_bytes(&src)
}
}
fn deserialize_human_readable_bytes_array<'de, const N: usize, D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<[u8; N], D::Error> {
use serde::Deserialize;
use std::borrow::Cow;
let s: Cow<str> = Cow::deserialize(deserializer)?;
let mut output = [0u8; N];
const_hex::decode_to_slice(s.as_ref(), &mut output)
.map(|_| output)
.map_err(<D::Error as serde::de::Error>::custom)
}
fn deserialize_human_unreadable_bytes_array<
'de,
const N: usize,
D: serde::Deserializer<'de>,
>(
deserializer: D,
) -> Result<[u8; N], D::Error> {
use serde::Deserialize;
<&[u8]>::deserialize(deserializer).and_then(|bytes| {
if bytes.len() != N {
Err(<D::Error as serde::de::Error>::invalid_length(
bytes.len(),
&format!("{N}").as_str(),
))
} else {
let mut output = [0u8; N];
output.copy_from_slice(bytes);
Ok(output)
}
})
}
pub fn deserialize_bytes_array<'de, const N: usize, D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<[u8; N], D::Error> {
if deserializer.is_human_readable() {
deserialize_human_readable_bytes_array(deserializer)
} else {
deserialize_human_unreadable_bytes_array(deserializer)
}
}
}