Humanize array ser/deser (#468)

* humanize array ser/deser

* split fns

* use `const-hex`

* fix fmt

* create `nomos-utils` crate

* Human serde committeeid (#478)

* Human readable serde for CommitteeId

* Deserialize bytes to string if human readable

* Don't allocate if possible in human serde bytes

---------

Co-authored-by: gusto <bacv@users.noreply.github.com>

Cargo.toml

@@ -15,6 +15,7 @@ members = [
     "nomos-da/full-replication",
     "nomos-http-api",
     "nomos-cli",
+    "nomos-utils",
     "nodes/nomos-node",
     "nodes/mixnode",
     "simulations",

consensus-engine/Cargo.toml

@@ -15,10 +15,11 @@ rand = "0.8"
 rand_chacha = "0.3"
 thiserror = "1"
 fraction = { version = "0.13" }
+nomos-utils = { path = "../nomos-utils", optional = true }
 
 [features]
 default = []
-serde = ["dep:serde"]
+serde = ["dep:serde", "nomos-utils/serde"]
 simulation = []
 
 [dev-dependencies]

consensus-engine/src/types/block_id.rs

@@ -1,8 +1,23 @@
 #[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq, Ord, PartialOrd)]
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-#[cfg_attr(feature = "serde", serde(transparent))]
 pub struct BlockId(pub(crate) [u8; 32]);
 
+#[cfg(feature = "serde")]
+impl serde::Serialize for BlockId {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        nomos_utils::serde::serialize_bytes_array(self.0, serializer)
+    }
+}
+
+#[cfg(feature = "serde")]
+impl<'de> serde::de::Deserialize<'de> for BlockId {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        nomos_utils::serde::deserialize_bytes_array(deserializer).map(Self)
+    }
+}
+
 impl BlockId {
     pub const fn new(val: [u8; 32]) -> Self {
         Self(val)

consensus-engine/src/types/committee.rs

@@ -3,9 +3,25 @@ use std::collections::BTreeSet;
 use crate::NodeId;
 
 #[derive(Debug, Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct CommitteeId(pub(crate) [u8; 32]);
 
+#[cfg(feature = "serde")]
+impl serde::Serialize for CommitteeId {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        nomos_utils::serde::serialize_bytes_array(self.0, serializer)
+    }
+}
+
+#[cfg(feature = "serde")]
+impl<'de> serde::de::Deserialize<'de> for CommitteeId {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        nomos_utils::serde::deserialize_bytes_array(deserializer).map(Self)
+    }
+}
+
 impl CommitteeId {
     pub const fn new(val: [u8; 32]) -> Self {
         Self(val)

consensus-engine/src/types/node_id.rs

@@ -1,8 +1,23 @@
 #[derive(Clone, Copy, Default, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)]
-#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
-#[cfg_attr(feature = "serde", serde(transparent))]
 pub struct NodeId(pub(crate) [u8; 32]);
 
+#[cfg(feature = "serde")]
+impl serde::Serialize for NodeId {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        nomos_utils::serde::serialize_bytes_array(self.0, serializer)
+    }
+}
+
+#[cfg(feature = "serde")]
+impl<'de> serde::de::Deserialize<'de> for NodeId {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        nomos_utils::serde::deserialize_bytes_array(deserializer).map(Self)
+    }
+}
+
 impl NodeId {
     pub const fn new(val: [u8; 32]) -> Self {
         Self(val)

nomos-services/consensus/src/lib.rs

@@ -1060,6 +1060,22 @@ pub struct CarnotInfo {
     pub committed_blocks: Vec<BlockId>,
 }
 
+async fn get_mempool_contents<Item, Key>(
+    mempool: OutboundRelay<MempoolMsg<Item, Key>>,
+) -> Result<Box<dyn Iterator<Item = Item> + Send>, tokio::sync::oneshot::error::RecvError> {
+    let (reply_channel, rx) = tokio::sync::oneshot::channel();
+
+    mempool
+        .send(MempoolMsg::View {
+            ancestor_hint: BlockId::zeros(),
+            reply_channel,
+        })
+        .await
+        .unwrap_or_else(|(e, _)| eprintln!("Could not get transactions from mempool {e}"));
+
+    rx.await
+}
+
 #[cfg(test)]
 mod tests {
     use consensus_engine::Block;
@@ -1098,26 +1114,10 @@ mod tests {
         eprintln!("{serialized}");
         assert_eq!(
             serialized,
-            r#"{"id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"current_view":1,"highest_voted_view":-1,"local_high_qc":{"view":0,"id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"safe_blocks":[[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],{"view":0,"parent_qc":{"Standard":{"view":0,"id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}},"leader_proof":{"LeaderId":{"leader_id":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}}}]],"last_view_timeout_qc":null,"committed_blocks":[[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]}"#
+            r#"{"id":"0x0000000000000000000000000000000000000000000000000000000000000000","current_view":1,"highest_voted_view":-1,"local_high_qc":{"view":0,"id":"0x0000000000000000000000000000000000000000000000000000000000000000"},"safe_blocks":[["0x0000000000000000000000000000000000000000000000000000000000000000",{"view":0,"parent_qc":{"Standard":{"view":0,"id":"0x0000000000000000000000000000000000000000000000000000000000000000"}},"leader_proof":{"LeaderId":{"leader_id":"0x0000000000000000000000000000000000000000000000000000000000000000"}}}]],"last_view_timeout_qc":null,"committed_blocks":["0x0000000000000000000000000000000000000000000000000000000000000000"]}"#
         );
 
         let deserialized: CarnotInfo = serde_json::from_str(&serialized).unwrap();
         assert_eq!(deserialized, info);
     }
 }
-
-async fn get_mempool_contents<Item, Key>(
-    mempool: OutboundRelay<MempoolMsg<Item, Key>>,
-) -> Result<Box<dyn Iterator<Item = Item> + Send>, tokio::sync::oneshot::error::RecvError> {
-    let (reply_channel, rx) = tokio::sync::oneshot::channel();
-
-    mempool
-        .send(MempoolMsg::View {
-            ancestor_hint: BlockId::zeros(),
-            reply_channel,
-        })
-        .await
-        .unwrap_or_else(|(e, _)| eprintln!("Could not get transactions from mempool {e}"));
-
-    rx.await
-}

nomos-utils/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "nomos-utils"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+serde = ["dep:serde"]
+
+[dependencies]
+const-hex = "1"
+serde = { version = "1.0", optional = true }

nomos-utils/src/lib.rs

@@ -0,0 +1,67 @@
+#[cfg(feature = "serde")]
+pub mod serde {
+    fn serialize_human_readable_bytes_array<const N: usize, S: serde::Serializer>(
+        src: [u8; N],
+        serializer: S,
+    ) -> Result<S::Ok, S::Error> {
+        use serde::Serialize;
+        const_hex::const_encode::<N, true>(&src)
+            .as_str()
+            .serialize(serializer)
+    }
+
+    pub fn serialize_bytes_array<const N: usize, S: serde::Serializer>(
+        src: [u8; N],
+        serializer: S,
+    ) -> Result<S::Ok, S::Error> {
+        if serializer.is_human_readable() {
+            serialize_human_readable_bytes_array(src, serializer)
+        } else {
+            serializer.serialize_bytes(&src)
+        }
+    }
+
+    fn deserialize_human_readable_bytes_array<'de, const N: usize, D: serde::Deserializer<'de>>(
+        deserializer: D,
+    ) -> Result<[u8; N], D::Error> {
+        use serde::Deserialize;
+        use std::borrow::Cow;
+        let s: Cow<str> = Cow::deserialize(deserializer)?;
+        let mut output = [0u8; N];
+        const_hex::decode_to_slice(s.as_ref(), &mut output)
+            .map(|_| output)
+            .map_err(<D::Error as serde::de::Error>::custom)
+    }
+
+    fn deserialize_human_unreadable_bytes_array<
+        'de,
+        const N: usize,
+        D: serde::Deserializer<'de>,
+    >(
+        deserializer: D,
+    ) -> Result<[u8; N], D::Error> {
+        use serde::Deserialize;
+        <&[u8]>::deserialize(deserializer).and_then(|bytes| {
+            if bytes.len() != N {
+                Err(<D::Error as serde::de::Error>::invalid_length(
+                    bytes.len(),
+                    &format!("{N}").as_str(),
+                ))
+            } else {
+                let mut output = [0u8; N];
+                output.copy_from_slice(bytes);
+                Ok(output)
+            }
+        })
+    }
+
+    pub fn deserialize_bytes_array<'de, const N: usize, D: serde::Deserializer<'de>>(
+        deserializer: D,
+    ) -> Result<[u8; N], D::Error> {
+        if deserializer.is_human_readable() {
+            deserialize_human_readable_bytes_array(deserializer)
+        } else {
+            deserialize_human_unreadable_bytes_array(deserializer)
+        }
+    }
+}