logos-co/nomos-node
1
0
Fork 0
mirror of https://github.com/logos-co/nomos-node.git synced 2026-08-31 11:31:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(da): Transaction dispersal in DA network (#1495)

Browse Source
This commit is contained in:
gusto
2025-08-22 16:21:32 +03:00
committed by GitHub
parent 1670a6db78
commit 58abb03b23
96 changed files with 2569 additions and 1793 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/code-check.yml
+1
View File
@@ -221,6 +221,7 @@ jobs:
if: failure()
with:
name: integration-test-artifacts
include-hidden-files: true
path: tests/.tmp*
build-docker:
dependencies_graph.dot
+19 -18
View File
@@ -28,12 +28,12 @@ digraph {
26 [ label = "nomos-sdp" shape = box]
27 [ label = "nomos-storage" shape = box]
28 [ label = "nomos-tracing" shape = box]
29 [ label = "nomos-da-verifier" shape = box]
30 [ label = "nomos-mempool" shape = box]
31 [ label = "nomos-tracing-service" shape = box]
32 [ label = "nomos-time" shape = box]
33 [ label = "nomos-da-dispersal" shape = box]
34 [ label = "nomos-da-indexer" shape = box]
29 [ label = "nomos-mempool" shape = box]
30 [ label = "nomos-tracing-service" shape = box]
31 [ label = "nomos-time" shape = box]
32 [ label = "nomos-da-dispersal" shape = box]
33 [ label = "nomos-da-indexer" shape = box]
34 [ label = "nomos-da-verifier" shape = box]
35 [ label = "nomos-node" shape = box]
36 [ label = "nomos-system-sig" shape = box]
37 [ label = "executor-http-client" shape = box]
@@ -57,12 +57,13 @@ digraph {
7 -> 35 [ ]
8 -> 9 [ ]
8 -> 3 [ ]
10 -> 32 [ ]
10 -> 33 [ ]
10 -> 34 [ ]
11 -> 12 [ ]
11 -> 4 [ ]
11 -> 30 [ ]
11 -> 32 [ ]
11 -> 29 [ ]
11 -> 31 [ ]
12 -> 14 [ ]
12 -> 17 [ ]
12 -> 18 [ ]
@@ -72,7 +73,7 @@ digraph {
15 -> 13 [ ]
16 -> 2 [ ]
17 -> 16 [ style = dotted]
19 -> 29 [ ]
19 -> 23 [ ]
20 -> 21 [ ]
20 -> 22 [ ]
21 -> 8 [ ]
@@ -91,15 +92,15 @@ digraph {
25 -> 18 [ ]
26 -> 3 [ ]
27 -> 3 [ ]
29 -> 23 [ ]
30 -> 19 [ ]
30 -> 30 [ color = blue]
30 -> 17 [ ]
30 -> 31 [ color = blue]
31 -> 28 [ ]
32 -> 0 [ ]
33 -> 30 [ ]
34 -> 11 [ ]
29 -> 19 [ ]
29 -> 29 [ color = blue]
29 -> 17 [ ]
29 -> 30 [ color = blue]
30 -> 28 [ ]
31 -> 0 [ ]
32 -> 23 [ ]
33 -> 11 [ ]
34 -> 29 [ ]
35 -> 10 [ ]
35 -> 36 [ ]
37 -> 24 [ ]
nodes/nomos-executor/config.yaml
+20 -8
View File
@@ -75,9 +75,6 @@ da_network:
num_of_subnets: 20
shares_retry_limit: 5
commitments_retry_limit: 5
refresh_interval:
secs: 30
nanos: 0
num_subnets: 2
membership:
subnetwork_size: 2
@@ -88,6 +85,9 @@ da_network:
api_adapter_settings:
api_port: 8722
is_secure: false
subnet_refresh_interval:
secs: 30
nanos: 0
da_dispersal:
backend:
encoder_settings:
@@ -95,25 +95,34 @@ da_dispersal:
with_cache: false
global_params_path: /tmp
dispersal_timeout: [20, 0]
mempool_strategy: !SampleSubnetworks
sample_threshold: 2
timeout: [10, 0]
cooldown: [0, 100]
da_indexer:
storage:
blob_storage_directory: ./
da_verifier:
verifier_settings:
share_verifier_settings:
sk: 67cee9fdc9c5160671b40da60b17153558496a12fb57804c1e64ef51f5a29dca
index:
- 1
- 0
global_params_path: ./tests/kzgrs/kzgrs_test_params
domain_size: 2
tx_verifier_settings: null
network_adapter_settings: null
storage_adapter_settings:
blob_storage_directory: ./
mempool_trigger_settings:
publish_threshold: 0.8
share_duration: [5, 0]
prune_duration: [30, 0]
prune_interval: [5, 0]
da_sampling:
share_verifier_settings:
sk: 67cee9fdc9c5160671b40da60b17153558496a12fb57804c1e64ef51f5a29dca
index:
- 1
- 0
global_params_path: ./tests/kzgrs/kzgrs_test_params
domain_size: 2
sampling_settings:
num_samples: 1
num_subnets: 2
@@ -323,6 +332,9 @@ storage:
mempool:
cl_pool_recovery_path: ./recovery/cl_mempool.json
da_pool_recovery_path: ./recovery/da_mempool.json
trigger_sampling_delay:
secs: 5
nanos: 0
membership:
backend:
settings_per_service: {}
nodes/nomos-executor/executor/src/api/backend.rs
+22 -83
View File
@@ -23,16 +23,15 @@ use nomos_core::{
DaVerifier as CoreDaVerifier,
},
header::HeaderId,
mantle::Transaction,
mantle::{SignedMantleTx, Transaction},
};
use nomos_da_dispersal::adapters::mempool::DaMempoolAdapter;
use nomos_da_network_core::SubnetworkId;
use nomos_da_network_service::{
backends::libp2p::executor::DaNetworkExecutorBackend, membership::MembershipAdapter,
storage::MembershipStorageAdapter,
};
use nomos_da_sampling::backend::DaSamplingServiceBackend;
use nomos_da_verifier::backend::VerifierBackend;
use nomos_da_verifier::{backend::VerifierBackend, mempool::DaMempoolAdapter};
use nomos_http_api_common::paths;
use nomos_libp2p::PeerId;
use nomos_mempool::{
@@ -75,7 +74,6 @@ pub struct AxumBackendSettings {
}
pub struct AxumBackend<
DaAttestation,
DaShare,
DaBlobInfo,
Memebership,
@@ -90,11 +88,11 @@ pub struct AxumBackend<
DaStorageConverter,
DispersalBackend,
DispersalNetworkAdapter,
DispersalMempoolAdapter,
Metadata,
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
VerifierMempoolAdapter,
TimeBackend,
ApiAdapter,
HttpStorageAdapter,
@@ -104,7 +102,6 @@ pub struct AxumBackend<
#[expect(clippy::allow_attributes_without_reason)]
#[expect(clippy::type_complexity)]
_phantom: core::marker::PhantomData<(
DaAttestation,
DaShare,
DaBlobInfo,
Memebership,
@@ -119,11 +116,11 @@ pub struct AxumBackend<
DaStorageConverter,
DispersalBackend,
DispersalNetworkAdapter,
DispersalMempoolAdapter,
Metadata,
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
VerifierMempoolAdapter,
TimeBackend,
ApiAdapter,
HttpStorageAdapter,
@@ -145,7 +142,6 @@ struct ApiDoc;
#[async_trait::async_trait]
impl<
DaAttestation,
DaShare,
DaBlobInfo,
Membership,
@@ -160,11 +156,11 @@ impl<
DaStorageConverter,
DispersalBackend,
DispersalNetworkAdapter,
DispersalMempoolAdapter,
Metadata,
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
VerifierMempoolAdapter,
TimeBackend,
ApiAdapter,
StorageAdapter,
@@ -172,7 +168,6 @@ impl<
RuntimeServiceId,
> Backend<RuntimeServiceId>
for AxumBackend<
DaAttestation,
DaShare,
DaBlobInfo,
Membership,
@@ -187,18 +182,17 @@ impl<
DaStorageConverter,
DispersalBackend,
DispersalNetworkAdapter,
DispersalMempoolAdapter,
Metadata,
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
VerifierMempoolAdapter,
TimeBackend,
ApiAdapter,
StorageAdapter,
SIZE,
>
where
DaAttestation: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
DaShare: Share + Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
<DaShare as Share>::BlobId: Clone + Serialize + DeserializeOwned + Send + Sync + 'static,
<DaShare as Share>::ShareIndex:
@@ -268,15 +262,12 @@ where
Serialize + for<'de> Deserialize<'de> + Ord + Debug + Send + Sync + 'static,
DaStorageSerializer: StorageSerde + Send + Sync + 'static,
<DaStorageSerializer as StorageSerde>::Error: Send + Sync,
DaStorageConverter: da::DaConverter<DaStorageBackend<DaStorageSerializer>, Share = DaShare>
DaStorageConverter: da::DaConverter<DaStorageBackend<DaStorageSerializer>, Share = DaShare, Tx = SignedMantleTx>
+ Send
+ Sync
+ 'static,
DispersalBackend: nomos_da_dispersal::backend::DispersalBackend<
NetworkAdapter = DispersalNetworkAdapter,
MempoolAdapter = DispersalMempoolAdapter,
Metadata = Metadata,
> + Send
DispersalBackend: nomos_da_dispersal::backend::DispersalBackend<NetworkAdapter = DispersalNetworkAdapter>
+ Send
+ Sync
+ 'static,
DispersalBackend::BlobId: Serialize,
@@ -285,7 +276,6 @@ where
SubnetworkId = Membership::NetworkId,
> + Send
+ 'static,
DispersalMempoolAdapter: DaMempoolAdapter + Send + 'static,
Metadata: DeserializeOwned + metadata::Metadata + Debug + Send + 'static,
SamplingBackend: DaSamplingServiceBackend<BlobId = <DaVerifiedBlobInfo as DispersedBlobInfo>::BlobId>
+ Send
@@ -304,6 +294,7 @@ where
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync + 'static,
SamplingStorage:
nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync + 'static,
VerifierMempoolAdapter: DaMempoolAdapter + Send + Sync + 'static,
TimeBackend: nomos_time::backends::TimeBackend + Send + 'static,
TimeBackend::Settings: Clone + Send + Sync,
ApiAdapter: nomos_da_network_service::api::ApiAdapter + Send + Sync + 'static,
@@ -322,9 +313,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -337,6 +325,7 @@ where
DaVerifierBackend,
DaStorageSerializer,
DaStorageConverter,
VerifierMempoolAdapter,
RuntimeServiceId,
>,
>
@@ -349,9 +338,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -382,9 +368,7 @@ where
RuntimeServiceId,
>,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
MockPool<HeaderId, Tx, <Tx as Transaction>::Hash>,
RuntimeServiceId,
>,
@@ -400,21 +384,11 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>
+ AsServiceId<
DaDispersal<
DispersalBackend,
DispersalNetworkAdapter,
DispersalMempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
>,
DaDispersal<DispersalBackend, DispersalNetworkAdapter, Membership, RuntimeServiceId>,
>,
{
type Error = hyper::Error;
@@ -437,15 +411,15 @@ where
wait_until_services_are_ready!(
&overwatch_handle,
Some(Duration::from_secs(60)),
Cryptarchia<_, _, _, _, _, _, _, _, _, _, SIZE>,
DaVerifier<_, _, _, _, _, _>,
DaIndexer<_, _, _, _, _, _, _, _, _, _, _, _, SIZE>,
Cryptarchia<_, _, _, _, _, _, _, SIZE>,
DaVerifier<_, _, _, _, _, _, _>,
DaIndexer<_, _, _, _, _, _, _, _, _, SIZE>,
nomos_da_network_service::NetworkService<_, _, _,_, _, _>,
nomos_network::NetworkService<_, _>,
DaStorageService<_, _>,
TxMempoolService<_, _, _, _, _, _, _>,
DaMempoolService<_, _, _, _, _, _, _, _, _>,
DaDispersal<_, _, _, _, _, _>
TxMempoolService<_, _, _, _, _>,
DaMempoolService<_, _, _, _, _, _>,
DaDispersal<_, _, _, _>
)
.await
}
@@ -477,27 +451,13 @@ where
.route(
paths::CL_METRICS,
routing::get(
cl_metrics::<
Tx,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
RuntimeServiceId,
>,
cl_metrics::<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>,
),
)
.route(
paths::CL_STATUS,
routing::post(
cl_status::<
Tx,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
RuntimeServiceId,
>,
cl_status::<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>,
),
)
.route(
@@ -509,9 +469,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -527,9 +484,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -540,12 +494,12 @@ where
paths::DA_ADD_SHARE,
routing::post(
add_share::<
DaAttestation,
DaShare,
DaVerifierNetwork,
DaVerifierBackend,
DaStorageSerializer,
DaStorageConverter,
VerifierMempoolAdapter,
RuntimeServiceId,
>,
),
@@ -561,9 +515,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -617,14 +568,7 @@ where
.route(
paths::MEMPOOL_ADD_TX,
routing::post(
add_tx::<
Tx,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
RuntimeServiceId,
>,
add_tx::<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>,
),
)
.route(
@@ -635,9 +579,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
),
@@ -648,9 +589,7 @@ where
disperse_data::<
DispersalBackend,
DispersalNetworkAdapter,
DispersalMempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
>,
),
nodes/nomos-executor/executor/src/api/handlers.rs
+7 -36
View File
@@ -1,18 +1,15 @@
use std::fmt::{Debug, Display};
use axum::{extract::State, response::Response, Json};
use kzgrs_backend::dispersal::Metadata;
use nomos_api::http::da::{self, DaDispersal};
use nomos_core::da::blob::metadata;
use nomos_da_dispersal::{
adapters::{mempool::DaMempoolAdapter, network::DispersalNetworkAdapter},
backend::DispersalBackend,
};
use nomos_da_dispersal::{adapters::network::DispersalNetworkAdapter, backend::DispersalBackend};
use nomos_da_network_core::SubnetworkId;
use nomos_http_api_common::{paths, types::DispersalRequest};
use nomos_libp2p::PeerId;
use nomos_node::make_request_and_return_response;
use overwatch::{overwatch::handle::OverwatchHandle, services::AsServiceId};
use serde::{de::DeserializeOwned, Serialize};
use serde::Serialize;
use subnetworks_assignations::MembershipHandler;
#[utoipa::path(
@@ -23,14 +20,7 @@ use subnetworks_assignations::MembershipHandler;
(status = 500, description = "Internal server error", body = String),
)
)]
pub async fn disperse_data<
Backend,
NetworkAdapter,
MempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
>(
pub async fn disperse_data<Backend, NetworkAdapter, Membership, RuntimeServiceId>(
State(handle): State<OverwatchHandle<RuntimeServiceId>>,
Json(dispersal_req): Json<DispersalRequest<Metadata>>,
) -> Response
@@ -41,38 +31,19 @@ where
+ Send
+ Sync
+ 'static,
Backend: DispersalBackend<
NetworkAdapter = NetworkAdapter,
MempoolAdapter = MempoolAdapter,
Metadata = Metadata,
> + Send
+ Sync
+ 'static,
Backend: DispersalBackend<NetworkAdapter = NetworkAdapter> + Send + Sync + 'static,
Backend::Settings: Clone + Send + Sync,
Backend::BlobId: Serialize,
NetworkAdapter: DispersalNetworkAdapter<SubnetworkId = Membership::NetworkId> + Send,
MempoolAdapter: DaMempoolAdapter,
Metadata: DeserializeOwned + metadata::Metadata + Debug + Send + 'static,
RuntimeServiceId: Debug
+ Sync
+ Display
+ AsServiceId<
DaDispersal<
Backend,
NetworkAdapter,
MempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
>,
>,
+ AsServiceId<DaDispersal<Backend, NetworkAdapter, Membership, RuntimeServiceId>>,
{
make_request_and_return_response!(da::disperse_data::<
Backend,
NetworkAdapter,
MempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
>(&handle, dispersal_req.data, dispersal_req.metadata))
>(&handle, dispersal_req.data))
}
nodes/nomos-executor/executor/src/lib.rs
+17 -77
View File
@@ -7,11 +7,11 @@ use nomos_blend_service::core::{
backends::libp2p::Libp2pBlendBackend as BlendBackend,
network::libp2p::Libp2pAdapter as BlendNetworkAdapter,
};
use nomos_core::{da::blob::info::DispersedBlobInfo, mantle::SignedMantleTx};
use nomos_core::mantle::SignedMantleTx;
use nomos_da_dispersal::{
adapters::{
mempool::kzgrs::KzgrsMempoolAdapter,
network::libp2p::Libp2pNetworkAdapter as DispersalNetworkAdapter,
wallet::mock::MockWalletAdapter as DispersalWalletAdapter,
},
backend::kzgrs::DispersalKZGRSBackend,
DispersalService,
@@ -29,15 +29,15 @@ use nomos_da_verifier::{
storage::adapters::rocksdb::RocksAdapter as VerifierStorageAdapter,
};
use nomos_libp2p::PeerId;
use nomos_mempool::backend::mockpool::MockPool;
#[cfg(feature = "tracing")]
use nomos_node::Tracing;
use nomos_node::{
generic_services::{
DaMembershipAdapter, DaMembershipStorageGeneric, MembershipService, SdpService,
VerifierMempoolAdapter,
},
BlobInfo, DaNetworkApiAdapter, HeaderId, MempoolNetworkAdapter, NetworkBackend,
NomosDaMembership, RocksBackend, SystemSig, Wire, MB16,
BlobInfo, DaNetworkApiAdapter, NetworkBackend, NomosDaMembership, RocksBackend, SystemSig,
Wire, MB16,
};
use nomos_time::backends::NtpTimeBackend;
use overwatch::derive_services;
@@ -68,29 +68,6 @@ pub(crate) type BlendEdgeService = nomos_blend_service::edge::BlendService<
pub(crate) type BlendService =
nomos_blend_service::BlendService<BlendCoreService, BlendEdgeService, RuntimeServiceId>;
type DispersalMempoolAdapter = KzgrsMempoolAdapter<
MempoolNetworkAdapter<BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId, RuntimeServiceId>,
MockPool<HeaderId, BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId>,
KzgrsSamplingBackend,
nomos_da_sampling::network::adapters::executor::Libp2pAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
SamplingStorageAdapter<DaShare, Wire, DaStorageConverter>,
KzgrsDaVerifier,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierStorageAdapter<DaShare, Wire, DaStorageConverter>,
RuntimeServiceId,
>;
pub(crate) type DaDispersalService = DispersalService<
DispersalKZGRSBackend<
DispersalNetworkAdapter<
@@ -100,7 +77,7 @@ pub(crate) type DaDispersalService = DispersalService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
DispersalMempoolAdapter,
DispersalWalletAdapter,
>,
DispersalNetworkAdapter<
NomosDaMembership,
@@ -109,9 +86,7 @@ pub(crate) type DaDispersalService = DispersalService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
DispersalMempoolAdapter,
NomosDaMembership,
kzgrs_backend::dispersal::Metadata,
RuntimeServiceId,
>;
@@ -123,13 +98,6 @@ pub(crate) type DaIndexerService = nomos_node::generic_services::DaIndexerServic
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
RuntimeServiceId,
>;
@@ -141,6 +109,7 @@ pub(crate) type DaVerifierService = nomos_node::generic_services::DaVerifierServ
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierMempoolAdapter<DaNetworkAdapter, RuntimeServiceId>,
RuntimeServiceId,
>;
@@ -152,13 +121,6 @@ pub(crate) type DaSamplingService = nomos_node::generic_services::DaSamplingServ
DaNetworkApiAdapter,
RuntimeServiceId,
>,
nomos_da_verifier::network::adapters::executor::Libp2pAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
RuntimeServiceId,
>;
@@ -179,34 +141,20 @@ pub(crate) type ClMempoolService = nomos_node::generic_services::TxMempoolServic
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
RuntimeServiceId,
>;
pub(crate) type DaMempoolService = nomos_node::generic_services::DaMempoolService<
nomos_da_sampling::network::adapters::executor::Libp2pAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
pub(crate) type DaNetworkAdapter = nomos_da_sampling::network::adapters::executor::Libp2pAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>;
pub(crate) type DaMempoolService =
nomos_node::generic_services::DaMempoolService<DaNetworkAdapter, RuntimeServiceId>;
pub(crate) type CryptarchiaService = nomos_node::generic_services::CryptarchiaService<
nomos_da_sampling::network::adapters::executor::Libp2pAdapter<
NomosDaMembership,
@@ -215,13 +163,6 @@ pub(crate) type CryptarchiaService = nomos_node::generic_services::CryptarchiaSe
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
RuntimeServiceId,
>;
@@ -232,7 +173,6 @@ pub(crate) type ApiStorageAdapter<StorageOp, RuntimeServiceId> =
pub(crate) type ApiService = nomos_api::ApiService<
AxumBackend<
(),
DaShare,
BlobInfo,
NomosDaMembership,
@@ -259,7 +199,7 @@ pub(crate) type ApiService = nomos_api::ApiService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
DispersalMempoolAdapter,
DispersalWalletAdapter,
>,
DispersalNetworkAdapter<
NomosDaMembership,
@@ -268,7 +208,6 @@ pub(crate) type ApiService = nomos_api::ApiService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
DispersalMempoolAdapter,
kzgrs_backend::dispersal::Metadata,
KzgrsSamplingBackend,
nomos_da_sampling::network::adapters::executor::Libp2pAdapter<
@@ -279,6 +218,7 @@ pub(crate) type ApiService = nomos_api::ApiService<
RuntimeServiceId,
>,
SamplingStorageAdapter<DaShare, Wire, DaStorageConverter>,
VerifierMempoolAdapter<DaNetworkAdapter, RuntimeServiceId>,
NtpTimeBackend,
DaNetworkApiAdapter,
ApiStorageAdapter<Wire, RuntimeServiceId>,
nodes/nomos-executor/executor/src/main.rs
+1
View File
@@ -92,6 +92,7 @@ async fn main() -> Result<()> {
id: <BlobInfo as DispersedBlobInfo>::blob_id,
},
recovery_path: config.mempool.da_pool_recovery_path,
trigger_sampling_delay: config.mempool.trigger_sampling_delay,
},
da_dispersal: config.da_dispersal,
da_network: config.da_network,
nodes/nomos-node/config.yaml
+20 -4
View File
@@ -74,9 +74,6 @@ da_network:
num_of_subnets: 20
shares_retry_limit: 5
commitments_retry_limit: 5
refresh_interval:
secs: 30
nanos: 0
membership:
replication_factor: 2
subnetwork_size: 2
@@ -86,21 +83,37 @@ da_network:
api_adapter_settings:
api_port: 8722
is_secure: false
subnet_refresh_interval:
secs: 30
nanos: 0
da_indexer:
storage:
blob_storage_directory: ./
da_verifier:
verifier_settings:
share_verifier_settings:
sk: 67cee9fdc9c5160671b40da60b17153558496a12fb57804c1e64ef51f5a29dca
index:
- 1
- 0
global_params_path: ./tests/kzgrs/kzgrs_test_params
domain_size: 2
tx_verifier_settings: null
network_adapter_settings: null
storage_adapter_settings:
blob_storage_directory: ./
mempool_trigger_settings:
publish_threshold: 0.8
share_duration: [5, 0]
prune_duration: [30, 0]
prune_interval: [5, 0]
da_sampling:
share_verifier_settings:
sk: 67cee9fdc9c5160671b40da60b17153558496a12fb57804c1e64ef51f5a29dca
index:
- 1
- 0
global_params_path: ./tests/kzgrs/kzgrs_test_params
domain_size: 2
sampling_settings:
num_samples: 1
num_subnets: 2
@@ -310,6 +323,9 @@ storage:
mempool:
cl_pool_recovery_path: ./recovery/cl_mempool.json
da_pool_recovery_path: ./recovery/da_mempool.json
trigger_sampling_delay:
secs: 5
nanos: 0
membership:
backend:
settings_per_service: {}
nodes/nomos-node/node/src/api/backend.rs
+21 -65
View File
@@ -23,7 +23,7 @@ use nomos_core::{
DaVerifier as CoreDaVerifier,
},
header::HeaderId,
mantle::Transaction,
mantle::{SignedMantleTx, Transaction},
};
use nomos_da_network_core::SubnetworkId;
use nomos_da_network_service::{
@@ -31,7 +31,7 @@ use nomos_da_network_service::{
storage::MembershipStorageAdapter,
};
use nomos_da_sampling::backend::DaSamplingServiceBackend;
use nomos_da_verifier::backend::VerifierBackend;
use nomos_da_verifier::{backend::VerifierBackend, mempool::DaMempoolAdapter};
use nomos_http_api_common::paths;
use nomos_libp2p::PeerId;
use nomos_mempool::{
@@ -74,7 +74,6 @@ pub struct AxumBackendSettings {
}
pub struct AxumBackend<
DaAttestation,
DaShare,
DaBlobInfo,
Membership,
@@ -90,13 +89,13 @@ pub struct AxumBackend<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
VerifierMempoolAdapter,
TimeBackend,
ApiAdapter,
HttpStorageAdapter,
const SIZE: usize,
> {
settings: AxumBackendSettings,
_attestation: core::marker::PhantomData<DaAttestation>,
_share: core::marker::PhantomData<DaShare>,
_certificate: core::marker::PhantomData<DaBlobInfo>,
_membership: core::marker::PhantomData<Membership>,
@@ -114,6 +113,7 @@ pub struct AxumBackend<
_api_adapter: core::marker::PhantomData<ApiAdapter>,
_storage_adapter: core::marker::PhantomData<HttpStorageAdapter>,
_da_membership: core::marker::PhantomData<(DaMembershipAdapter, DaMembershipStorage)>,
_verifier_mempool_adapter: core::marker::PhantomData<VerifierMempoolAdapter>,
}
#[derive(OpenApi)]
@@ -131,7 +131,6 @@ struct ApiDoc;
#[async_trait::async_trait]
impl<
DaAttestation,
DaShare,
DaBlobInfo,
Membership,
@@ -147,6 +146,7 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
VerifierMempoolAdapter,
TimeBackend,
ApiAdapter,
StorageAdapter,
@@ -154,7 +154,6 @@ impl<
RuntimeServiceId,
> Backend<RuntimeServiceId>
for AxumBackend<
DaAttestation,
DaShare,
DaBlobInfo,
Membership,
@@ -170,13 +169,13 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
VerifierMempoolAdapter,
TimeBackend,
ApiAdapter,
StorageAdapter,
SIZE,
>
where
DaAttestation: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
DaShare: Share + Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
<DaShare as Share>::BlobId: Clone + Serialize + DeserializeOwned + Send + Sync + 'static,
<DaShare as Share>::ShareIndex: Serialize + DeserializeOwned + Send + Sync + 'static,
@@ -257,11 +256,14 @@ where
SamplingStorage:
nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync + 'static,
DaVerifierNetwork::Settings: Clone,
VerifierMempoolAdapter: DaMempoolAdapter + Send + Sync + 'static,
TimeBackend: nomos_time::backends::TimeBackend + Send + 'static,
TimeBackend::Settings: Clone + Send + Sync,
ApiAdapter: nomos_da_network_service::api::ApiAdapter + Send + Sync + 'static,
DaStorageConverter:
DaConverter<DaStorageBackend<DaStorageSerializer>, Share = DaShare> + Send + Sync + 'static,
DaStorageConverter: DaConverter<DaStorageBackend<DaStorageSerializer>, Share = DaShare, Tx = SignedMantleTx>
+ Send
+ Sync
+ 'static,
StorageAdapter:
storage::StorageAdapter<DaStorageSerializer, RuntimeServiceId> + Send + Sync + 'static,
RuntimeServiceId: Debug
@@ -277,9 +279,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -292,6 +291,7 @@ where
DaVerifierBackend,
DaStorageSerializer,
DaStorageConverter,
VerifierMempoolAdapter,
RuntimeServiceId,
>,
>
@@ -304,9 +304,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -337,9 +334,7 @@ where
RuntimeServiceId,
>,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
MockPool<HeaderId, Tx, <Tx as Transaction>::Hash>,
RuntimeServiceId,
>,
@@ -355,9 +350,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>,
@@ -371,7 +363,6 @@ where
{
Ok(Self {
settings,
_attestation: core::marker::PhantomData,
_share: core::marker::PhantomData,
_certificate: core::marker::PhantomData,
_membership: core::marker::PhantomData,
@@ -389,6 +380,7 @@ where
_api_adapter: core::marker::PhantomData,
_storage_adapter: core::marker::PhantomData,
_da_membership: core::marker::PhantomData,
_verifier_mempool_adapter: core::marker::PhantomData,
})
}
@@ -407,18 +399,15 @@ where
_,
_,
_,
_,
_,
_,
SIZE,
>,
DaVerifier<_, _, _, _, _, _>,
DaIndexer<_, _, _, _, _, _, _, _, _, _, _, _, SIZE>,
DaVerifier<_, _, _, _, _, _, _>,
DaIndexer<_, _, _, _, _, _, _, _, _, SIZE>,
nomos_da_network_service::NetworkService<_, _, _, _, _, _>,
nomos_network::NetworkService<_, _>,
DaStorageService<_, _>,
TxMempoolService<_, _, _, _, _, _, _>,
DaMempoolService<_, _, _, _, _, _, _, _, _>
TxMempoolService<_, _, _, _, _>,
DaMempoolService<_, _, _, _, _, _>
)
.await?;
Ok(())
@@ -451,27 +440,13 @@ where
.route(
paths::CL_METRICS,
routing::get(
cl_metrics::<
Tx,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
RuntimeServiceId,
>,
cl_metrics::<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>,
),
)
.route(
paths::CL_STATUS,
routing::post(
cl_status::<
Tx,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
RuntimeServiceId,
>,
cl_status::<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>,
),
)
.route(
@@ -483,9 +458,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -501,9 +473,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -514,12 +483,12 @@ where
paths::DA_ADD_SHARE,
routing::post(
add_share::<
DaAttestation,
DaShare,
DaVerifierNetwork,
DaVerifierBackend,
DaStorageSerializer,
DaStorageConverter,
VerifierMempoolAdapter,
RuntimeServiceId,
>,
),
@@ -535,9 +504,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -594,14 +560,7 @@ where
.route(
paths::MEMPOOL_ADD_TX,
routing::post(
add_tx::<
Tx,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
RuntimeServiceId,
>,
add_tx::<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>,
),
)
.route(
@@ -612,9 +571,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
),
nodes/nomos-node/node/src/api/handlers.rs
+16 -131
View File
@@ -23,7 +23,7 @@ use nomos_core::{
BlobId, DaVerifier as CoreDaVerifier,
},
header::HeaderId,
mantle::Transaction,
mantle::{SignedMantleTx, Transaction},
};
use nomos_da_messages::http::da::{
DASharesCommitmentsRequest, DaSamplingRequest, GetRangeReq, GetSharesRequest,
@@ -32,7 +32,7 @@ use nomos_da_network_service::{
api::ApiAdapter as ApiAdapterTrait, backends::NetworkBackend, NetworkService,
};
use nomos_da_sampling::backend::DaSamplingServiceBackend;
use nomos_da_verifier::backend::VerifierBackend;
use nomos_da_verifier::{backend::VerifierBackend, mempool::DaMempoolAdapter};
use nomos_http_api_common::paths;
use nomos_libp2p::PeerId;
use nomos_mempool::{
@@ -75,14 +75,7 @@ macro_rules! make_request_and_return_response {
(status = 500, description = "Internal server error", body = String),
)
)]
pub async fn cl_metrics<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>(
pub async fn cl_metrics<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>(
State(handle): State<OverwatchHandle<RuntimeServiceId>>,
) -> Response
where
@@ -91,32 +84,18 @@ where
Ord + Debug + Send + Sync + Serialize + for<'de> Deserialize<'de> + 'static,
SamplingNetworkAdapter:
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
VerifierNetworkAdapter:
nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
VerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
RuntimeServiceId: Debug
+ Send
+ Sync
+ Display
+ 'static
+ AsServiceId<
ClMempoolService<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>,
>,
+ AsServiceId<ClMempoolService<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>>,
{
make_request_and_return_response!(cl::cl_mempool_metrics::<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>(&handle))
}
@@ -129,14 +108,7 @@ where
(status = 500, description = "Internal server error", body = String),
)
)]
pub async fn cl_status<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>(
pub async fn cl_status<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>(
State(handle): State<OverwatchHandle<RuntimeServiceId>>,
Json(items): Json<Vec<<Tx as Transaction>::Hash>>,
) -> Response
@@ -145,32 +117,18 @@ where
<Tx as Transaction>::Hash: Serialize + DeserializeOwned + Ord + Debug + Send + Sync + 'static,
SamplingNetworkAdapter:
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
VerifierNetworkAdapter:
nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
VerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
RuntimeServiceId: Debug
+ Send
+ Sync
+ Display
+ 'static
+ AsServiceId<
ClMempoolService<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>,
>,
+ AsServiceId<ClMempoolService<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>>,
{
make_request_and_return_response!(cl::cl_mempool_status::<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>(&handle, items))
}
@@ -194,9 +152,6 @@ pub async fn cryptarchia_info<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
const SIZE: usize,
@@ -214,11 +169,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -232,9 +182,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -247,9 +194,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -270,9 +214,6 @@ pub async fn cryptarchia_headers<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
const SIZE: usize,
@@ -291,11 +232,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -309,9 +245,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -325,9 +258,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -342,12 +272,11 @@ where
(status = 500, description = "Internal server error", body = String),
)
)]
pub async fn add_share<A, S, N, VB, SS, StorageConverter, RuntimeServiceId>(
pub async fn add_share<S, N, VB, SS, StorageConverter, VerifierMempoolAdapter, RuntimeServiceId>(
State(handle): State<OverwatchHandle<RuntimeServiceId>>,
Json(share): Json<S>,
) -> Response
where
A: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
S: Share + Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
<S as Share>::BlobId: Clone + Send + Sync + 'static,
<S as Share>::ShareIndex: Clone + Hash + Eq + Send + Sync + 'static,
@@ -359,20 +288,24 @@ where
<VB as VerifierBackend>::Settings: Clone,
<VB as CoreDaVerifier>::Error: Error,
SS: StorageSerde + Send + Sync + 'static,
StorageConverter: DaConverter<DaStorageBackend<SS>, Share = S> + Send + Sync + 'static,
StorageConverter:
DaConverter<DaStorageBackend<SS>, Share = S, Tx = SignedMantleTx> + Send + Sync + 'static,
VerifierMempoolAdapter: DaMempoolAdapter + Send + Sync + 'static,
RuntimeServiceId: Debug
+ Sync
+ Display
+ 'static
+ AsServiceId<DaVerifier<S, N, VB, SS, StorageConverter, RuntimeServiceId>>,
+ AsServiceId<
DaVerifier<S, N, VB, SS, StorageConverter, VerifierMempoolAdapter, RuntimeServiceId>,
>,
{
make_request_and_return_response!(da::add_share::<
A,
S,
N,
VB,
SS,
StorageConverter,
VerifierMempoolAdapter,
RuntimeServiceId,
>(&handle, share))
}
@@ -393,9 +326,6 @@ pub async fn get_range<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
const SIZE: usize,
@@ -440,11 +370,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -460,9 +385,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -477,9 +399,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -949,14 +868,7 @@ where
(status = 500, description = "Internal server error", body = String),
)
)]
pub async fn add_tx<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>(
pub async fn add_tx<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>(
State(handle): State<OverwatchHandle<RuntimeServiceId>>,
Json(tx): Json<Tx>,
) -> Response
@@ -966,10 +878,7 @@ where
Ord + Debug + Send + Sync + Serialize + for<'de> Deserialize<'de> + 'static,
SamplingNetworkAdapter:
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
VerifierNetworkAdapter:
nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
VerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
RuntimeServiceId: Debug
+ Sync
+ Send
@@ -979,9 +888,7 @@ where
TxMempoolService<
MempoolNetworkAdapter<Tx, <Tx as Transaction>::Hash, RuntimeServiceId>,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
MockPool<HeaderId, Tx, <Tx as Transaction>::Hash>,
RuntimeServiceId,
>,
@@ -991,9 +898,7 @@ where
Libp2pNetworkBackend,
MempoolNetworkAdapter<Tx, <Tx as Transaction>::Hash, RuntimeServiceId>,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
Tx,
<Tx as Transaction>::Hash,
RuntimeServiceId,
@@ -1008,16 +913,7 @@ where
(status = 500, description = "Internal server error", body = String),
)
)]
pub async fn add_blob_info<
B,
SamplingBackend,
SamplingAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>(
pub async fn add_blob_info<B, SamplingBackend, SamplingAdapter, SamplingStorage, RuntimeServiceId>(
State(handle): State<OverwatchHandle<RuntimeServiceId>>,
Json(blob_info): Json<B>,
) -> Response
@@ -1040,11 +936,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + 'static,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
RuntimeServiceId: Debug
+ Sync
+ Display
@@ -1056,9 +947,6 @@ where
SamplingBackend,
SamplingAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>,
@@ -1071,9 +959,6 @@ where
SamplingBackend,
SamplingAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>(&handle, blob_info, DispersedBlobInfo::blob_id))
}
nodes/nomos-node/node/src/config/mempool.rs
+2 -1
View File
@@ -1,4 +1,4 @@
use std::path::PathBuf;
use std::{path::PathBuf, time::Duration};
use serde::{Deserialize, Serialize};
@@ -6,4 +6,5 @@ use serde::{Deserialize, Serialize};
pub struct MempoolConfig {
pub cl_pool_recovery_path: PathBuf,
pub da_pool_recovery_path: PathBuf,
pub trigger_sampling_delay: Duration,
}
nodes/nomos-node/node/src/generic_services.rs
+88 -127
View File
@@ -16,7 +16,7 @@ use nomos_da_network_service::{
use nomos_da_sampling::{
backend::kzgrs::KzgrsSamplingBackend, storage::adapters::rocksdb::converter::DaStorageConverter,
};
use nomos_da_verifier::backend::kzgrs::KzgrsDaVerifier;
use nomos_da_verifier::{backend::kzgrs::KzgrsDaVerifier, mempool::kzgrs::KzgrsMempoolAdapter};
use nomos_libp2p::PeerId;
use nomos_membership::{adapters::sdp::LedgerSdpAdapter, backends::mock::MockMembershipBackend};
use nomos_mempool::backend::mockpool::MockPool;
@@ -32,7 +32,7 @@ use nomos_time::backends::NtpTimeBackend;
use crate::{Wire, MB16};
pub type TxMempoolService<SamplingNetworkAdapter, VerifierNetworkAdapter, RuntimeServiceId> =
pub type TxMempoolService<SamplingNetworkAdapter, RuntimeServiceId> =
nomos_mempool::TxMempoolService<
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
SignedMantleTx,
@@ -40,17 +40,11 @@ pub type TxMempoolService<SamplingNetworkAdapter, VerifierNetworkAdapter, Runtim
RuntimeServiceId,
>,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
nomos_da_sampling::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
nomos_da_verifier::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
MockPool<HeaderId, SignedMantleTx, <SignedMantleTx as Transaction>::Hash>,
RuntimeServiceId,
>;
@@ -73,57 +67,54 @@ pub type BlendService<RuntimeServiceId> = nomos_blend_service::BlendService<
RuntimeServiceId,
>;
pub type DaIndexerService<SamplingAdapter, VerifierNetwork, RuntimeServiceId> =
nomos_da_indexer::DataIndexerService<
// Indexer specific.
DaShare,
nomos_da_indexer::storage::adapters::rocksdb::RocksAdapter<
Wire,
BlobInfo,
DaStorageConverter,
>,
CryptarchiaConsensusAdapter<SignedMantleTx, BlobInfo>,
// Cryptarchia specific, should be the same as in `Cryptarchia` type above.
chain_service::network::adapters::libp2p::LibP2pAdapter<
SignedMantleTx,
BlobInfo,
RuntimeServiceId,
>,
BlendService<RuntimeServiceId>,
MockPool<HeaderId, SignedMantleTx, <SignedMantleTx as Transaction>::Hash>,
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
SignedMantleTx,
<SignedMantleTx as Transaction>::Hash,
RuntimeServiceId,
>,
MockPool<HeaderId, BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId>,
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
BlobInfo,
<BlobInfo as DispersedBlobInfo>::BlobId,
RuntimeServiceId,
>,
nomos_core::mantle::select::FillSize<MB16, SignedMantleTx>,
nomos_core::da::blob::select::FillSize<MB16, BlobInfo>,
RocksBackend<Wire>,
KzgrsSamplingBackend,
SamplingAdapter,
nomos_da_sampling::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
KzgrsDaVerifier,
VerifierNetwork,
nomos_da_verifier::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
NtpTimeBackend,
pub type DaIndexerService<SamplingAdapter, RuntimeServiceId> = nomos_da_indexer::DataIndexerService<
// Indexer specific.
DaShare,
nomos_da_indexer::storage::adapters::rocksdb::RocksAdapter<Wire, BlobInfo, DaStorageConverter>,
CryptarchiaConsensusAdapter<SignedMantleTx, BlobInfo>,
// Cryptarchia specific, should be the same as in `Cryptarchia` type above.
chain_service::network::adapters::libp2p::LibP2pAdapter<
SignedMantleTx,
BlobInfo,
RuntimeServiceId,
>;
>,
BlendService<RuntimeServiceId>,
MockPool<HeaderId, SignedMantleTx, <SignedMantleTx as Transaction>::Hash>,
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
SignedMantleTx,
<SignedMantleTx as Transaction>::Hash,
RuntimeServiceId,
>,
MockPool<HeaderId, BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId>,
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
BlobInfo,
<BlobInfo as DispersedBlobInfo>::BlobId,
RuntimeServiceId,
>,
nomos_core::mantle::select::FillSize<MB16, SignedMantleTx>,
nomos_core::da::blob::select::FillSize<MB16, BlobInfo>,
RocksBackend<Wire>,
KzgrsSamplingBackend,
SamplingAdapter,
nomos_da_sampling::storage::adapters::rocksdb::RocksAdapter<DaShare, Wire, DaStorageConverter>,
NtpTimeBackend,
RuntimeServiceId,
>;
pub type DaVerifierService<VerifierAdapter, RuntimeServiceId> =
pub type VerifierMempoolAdapter<NetworkAdapter, RuntimeServiceId> = KzgrsMempoolAdapter<
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
BlobInfo,
<BlobInfo as DispersedBlobInfo>::BlobId,
RuntimeServiceId,
>,
MockPool<HeaderId, BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId>,
KzgrsSamplingBackend,
NetworkAdapter,
nomos_da_sampling::storage::adapters::rocksdb::RocksAdapter<DaShare, Wire, DaStorageConverter>,
RuntimeServiceId,
>;
pub type DaVerifierService<VerifierAdapter, MempoolAdapter, RuntimeServiceId> =
nomos_da_verifier::DaVerifierService<
KzgrsDaVerifier,
VerifierAdapter,
@@ -132,10 +123,11 @@ pub type DaVerifierService<VerifierAdapter, RuntimeServiceId> =
Wire,
DaStorageConverter,
>,
MempoolAdapter,
RuntimeServiceId,
>;
pub type DaSamplingService<SamplingAdapter, VerifierNetworkAdapter, RuntimeServiceId> =
pub type DaSamplingService<SamplingAdapter, RuntimeServiceId> =
nomos_da_sampling::DaSamplingService<
KzgrsSamplingBackend,
SamplingAdapter,
@@ -144,81 +136,50 @@ pub type DaSamplingService<SamplingAdapter, VerifierNetworkAdapter, RuntimeServi
Wire,
DaStorageConverter,
>,
KzgrsDaVerifier,
VerifierNetworkAdapter,
nomos_da_verifier::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
RuntimeServiceId,
>;
pub type DaMempoolService<DaSamplingNetwork, VerifierNetwork, RuntimeServiceId> =
nomos_mempool::DaMempoolService<
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
BlobInfo,
<BlobInfo as DispersedBlobInfo>::BlobId,
RuntimeServiceId,
>,
MockPool<HeaderId, BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId>,
KzgrsSamplingBackend,
DaSamplingNetwork,
nomos_da_sampling::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
KzgrsDaVerifier,
VerifierNetwork,
nomos_da_verifier::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
pub type DaMempoolService<DaSamplingNetwork, RuntimeServiceId> = nomos_mempool::DaMempoolService<
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
BlobInfo,
<BlobInfo as DispersedBlobInfo>::BlobId,
RuntimeServiceId,
>;
>,
MockPool<HeaderId, BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId>,
KzgrsSamplingBackend,
DaSamplingNetwork,
nomos_da_sampling::storage::adapters::rocksdb::RocksAdapter<DaShare, Wire, DaStorageConverter>,
RuntimeServiceId,
>;
pub type CryptarchiaService<SamplingAdapter, VerifierNetwork, RuntimeServiceId> =
CryptarchiaConsensus<
chain_service::network::adapters::libp2p::LibP2pAdapter<
SignedMantleTx,
BlobInfo,
RuntimeServiceId,
>,
BlendService<RuntimeServiceId>,
MockPool<HeaderId, SignedMantleTx, <SignedMantleTx as Transaction>::Hash>,
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
SignedMantleTx,
<SignedMantleTx as Transaction>::Hash,
RuntimeServiceId,
>,
MockPool<HeaderId, BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId>,
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
BlobInfo,
<BlobInfo as DispersedBlobInfo>::BlobId,
RuntimeServiceId,
>,
nomos_core::mantle::select::FillSize<MB16, SignedMantleTx>,
nomos_core::da::blob::select::FillSize<MB16, BlobInfo>,
RocksBackend<Wire>,
KzgrsSamplingBackend,
SamplingAdapter,
nomos_da_sampling::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
KzgrsDaVerifier,
VerifierNetwork,
nomos_da_verifier::storage::adapters::rocksdb::RocksAdapter<
DaShare,
Wire,
DaStorageConverter,
>,
NtpTimeBackend,
pub type CryptarchiaService<SamplingAdapter, RuntimeServiceId> = CryptarchiaConsensus<
chain_service::network::adapters::libp2p::LibP2pAdapter<
SignedMantleTx,
BlobInfo,
RuntimeServiceId,
>;
>,
BlendService<RuntimeServiceId>,
MockPool<HeaderId, SignedMantleTx, <SignedMantleTx as Transaction>::Hash>,
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
SignedMantleTx,
<SignedMantleTx as Transaction>::Hash,
RuntimeServiceId,
>,
MockPool<HeaderId, BlobInfo, <BlobInfo as DispersedBlobInfo>::BlobId>,
nomos_mempool::network::adapters::libp2p::Libp2pAdapter<
BlobInfo,
<BlobInfo as DispersedBlobInfo>::BlobId,
RuntimeServiceId,
>,
nomos_core::mantle::select::FillSize<MB16, SignedMantleTx>,
nomos_core::da::blob::select::FillSize<MB16, BlobInfo>,
RocksBackend<Wire>,
KzgrsSamplingBackend,
SamplingAdapter,
nomos_da_sampling::storage::adapters::rocksdb::RocksAdapter<DaShare, Wire, DaStorageConverter>,
NtpTimeBackend,
RuntimeServiceId,
>;
pub type MembershipService<RuntimeServiceId> = nomos_membership::MembershipService<
MembershipBackend,
nodes/nomos-node/node/src/lib.rs
+11 -44
View File
@@ -4,6 +4,7 @@ pub mod generic_services;
use bytes::Bytes;
use color_eyre::eyre::Result;
use generic_services::VerifierMempoolAdapter;
use kzgrs_backend::common::share::DaShare;
pub use kzgrs_backend::dispersal::BlobInfo;
pub use nomos_blend_service::core::{
@@ -117,13 +118,6 @@ pub(crate) type DaIndexerService = generic_services::DaIndexerService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
RuntimeServiceId,
>;
@@ -135,6 +129,7 @@ pub(crate) type DaVerifierService = generic_services::DaVerifierService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierMempoolAdapter<DaNetworkAdapter, RuntimeServiceId>,
RuntimeServiceId,
>;
@@ -146,13 +141,6 @@ pub(crate) type DaSamplingService = generic_services::DaSamplingService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
RuntimeServiceId,
>;
@@ -173,34 +161,20 @@ pub(crate) type ClMempoolService = generic_services::TxMempoolService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
RuntimeServiceId,
>;
pub(crate) type DaMempoolService = generic_services::DaMempoolService<
nomos_da_sampling::network::adapters::validator::Libp2pAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
pub(crate) type DaNetworkAdapter = nomos_da_sampling::network::adapters::validator::Libp2pAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>;
pub(crate) type DaMempoolService =
generic_services::DaMempoolService<DaNetworkAdapter, RuntimeServiceId>;
pub(crate) type CryptarchiaService = generic_services::CryptarchiaService<
nomos_da_sampling::network::adapters::validator::Libp2pAdapter<
NomosDaMembership,
@@ -209,13 +183,6 @@ pub(crate) type CryptarchiaService = generic_services::CryptarchiaService<
DaNetworkApiAdapter,
RuntimeServiceId,
>,
VerifierNetworkAdapter<
NomosDaMembership,
DaMembershipAdapter<RuntimeServiceId>,
DaMembershipStorage,
DaNetworkApiAdapter,
RuntimeServiceId,
>,
RuntimeServiceId,
>;
@@ -226,7 +193,6 @@ pub(crate) type ApiStorageAdapter<StorageOp, RuntimeServiceId> =
pub(crate) type ApiService = nomos_api::ApiService<
AxumBackend<
(),
DaShare,
BlobInfo,
NomosDaMembership,
@@ -254,6 +220,7 @@ pub(crate) type ApiService = nomos_api::ApiService<
RuntimeServiceId,
>,
SamplingStorageAdapter<DaShare, Wire, DaStorageConverter>,
VerifierMempoolAdapter<DaNetworkAdapter, RuntimeServiceId>,
NtpTimeBackend,
DaNetworkApiAdapter,
ApiStorageAdapter<Wire, RuntimeServiceId>,
nodes/nomos-node/node/src/main.rs
+1
View File
@@ -58,6 +58,7 @@ async fn main() -> Result<()> {
id: <BlobInfo as DispersedBlobInfo>::blob_id,
},
recovery_path: config.mempool.da_pool_recovery_path,
trigger_sampling_delay: config.mempool.trigger_sampling_delay,
},
da_network: config.da_network,
da_indexer: config.da_indexer,
nomos-core/chain-defs/src/da/mod.rs
+10 -1
View File
@@ -2,6 +2,8 @@ pub mod blob;
use blob::Share;
use crate::mantle::ops::channel::Ed25519PublicKey;
pub type BlobId = [u8; 32];
pub trait DaEncoder {
@@ -27,5 +29,12 @@ pub trait DaDispersal {
type EncodedData;
type Error;
async fn disperse(&self, encoded_data: Self::EncodedData) -> Result<(), Self::Error>;
async fn disperse_shares(&self, encoded_data: Self::EncodedData) -> Result<(), Self::Error>;
async fn disperse_tx(
&self,
blob_id: BlobId,
num_columns: usize,
original_size: usize,
signer: Ed25519PublicKey,
) -> Result<(), Self::Error>;
}
nomos-core/chain-defs/src/mantle/ops/blob.rs
+33
View File
@@ -0,0 +1,33 @@
use serde::{Deserialize, Serialize};
pub(crate) const DA_COLUMNS: u64 = 1024;
pub(crate) const DA_ELEMENT_SIZE: u64 = 32;
use crate::mantle::{
gas::Gas,
ops::{ChannelId, Ed25519PublicKey},
tx::TxHash,
};
pub type BlobId = [u8; 32];
#[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)]
pub struct BlobOp {
pub channel: ChannelId,
pub blob: BlobId,
pub blob_size: u64,
pub da_storage_gas_price: Gas,
pub after_tx: Option<TxHash>,
pub signer: Ed25519PublicKey,
}
impl BlobOp {
#[must_use]
pub fn as_sign_bytes(&self) -> bytes::Bytes {
let mut buff = bytes::BytesMut::new();
buff.extend_from_slice(&self.channel.to_be_bytes());
buff.extend_from_slice(&self.blob);
buff.extend_from_slice(&self.signer);
buff.freeze()
}
}
nomos-core/chain-defs/src/mantle/ops/channel/blob.rs
+9
View File
@@ -30,4 +30,13 @@ impl BlobOp {
hasher.update(self.signer.as_ref());
MsgId(hasher.finalize().into())
}
#[must_use]
pub fn as_sign_bytes(&self) -> bytes::Bytes {
let mut buff = bytes::BytesMut::new();
buff.extend_from_slice(self.channel.as_ref());
buff.extend_from_slice(&self.blob);
buff.extend_from_slice(self.signer.as_ref());
buff.freeze()
}
}
nomos-core/chain-defs/src/mantle/ops/mod.rs
+3
View File
@@ -114,6 +114,9 @@ impl Op {
let mut buff = bytes::BytesMut::new();
buff.extend_from_slice(&[self.opcode()]);
// TODO: add ops payload
if let Self::ChannelBlob(blob_op) = self {
buff.extend_from_slice(&blob_op.as_sign_bytes());
}
buff.freeze()
}
nomos-da/network/core/src/behaviour/executor.rs
+1 -1
View File
@@ -78,7 +78,7 @@ where
membership.clone(),
addressbook.clone(),
),
validator_dispersal: DispersalValidatorBehaviour::new(membership.clone()),
validator_dispersal: DispersalValidatorBehaviour::new(peer_id, membership.clone()),
replication: ReplicationBehaviour::new(replication_config, peer_id, membership),
balancer: ConnectionBalancerBehaviour::new(addressbook, balancer),
monitor: ConnectionMonitorBehaviour::new(monitor, redial_cooldown),
nomos-da/network/core/src/behaviour/validator.rs
+1 -1
View File
@@ -69,7 +69,7 @@ where
subnets_config,
refresh_signal,
),
dispersal: DispersalValidatorBehaviour::new(membership.clone()),
dispersal: DispersalValidatorBehaviour::new(peer_id, membership.clone()),
replication: ReplicationBehaviour::new(replication_config, peer_id, membership),
balancer: ConnectionBalancerBehaviour::new(addressbook, balancer),
monitor: ConnectionMonitorBehaviour::new(monitor, redial_cooldown),
nomos-da/network/core/src/protocols/dispersal/executor/behaviour.rs
+78 -63
View File
@@ -19,9 +19,8 @@ use libp2p::{
Multiaddr, PeerId, Stream,
};
use libp2p_stream::{Control, OpenStreamError};
use nomos_core::{da::BlobId, wire};
use nomos_core::{da::BlobId, mantle::SignedMantleTx, wire};
use nomos_da_messages::{
common::Share,
dispersal,
packing::{pack_to_writer, unpack_from_reader},
};
@@ -53,6 +52,8 @@ pub enum DispersalError {
subnetwork_id: SubnetworkId,
error: dispersal::DispersalError,
},
#[error("Message has no blob id")]
NoBlobId,
#[error("Error dialing peer [{peer_id}]: {error}")]
OpenStreamError {
peer_id: PeerId,
@@ -70,7 +71,7 @@ impl DispersalError {
error: dispersal::DispersalError { blob_id, .. },
..
} => Some(*blob_id),
Self::OpenStreamError { .. } => None,
Self::NoBlobId | Self::OpenStreamError { .. } => None,
}
}
@@ -80,7 +81,7 @@ impl DispersalError {
Self::Io { subnetwork_id, .. }
| Self::Serialization { subnetwork_id, .. }
| Self::Protocol { subnetwork_id, .. } => Some(*subnetwork_id),
Self::OpenStreamError { .. } => None,
Self::NoBlobId | Self::OpenStreamError { .. } => None,
}
}
@@ -123,6 +124,7 @@ impl Clone for DispersalError {
subnetwork_id: *subnetwork_id,
error: error.clone(),
},
Self::NoBlobId => Self::NoBlobId,
Self::OpenStreamError { peer_id, error } => Self::OpenStreamError {
peer_id: *peer_id,
error: match error {
@@ -183,9 +185,10 @@ where
/// Address book handler to get addresses of peers
addressbook: Addressbook,
/// Pending blobs that need to be dispersed by `PeerId`
to_disperse: HashMap<PeerId, VecDeque<(Membership::NetworkId, DaShare)>>,
to_disperse: HashMap<PeerId, VecDeque<(Membership::NetworkId, dispersal::DispersalRequest)>>,
/// Pending blobs from disconnected networks
disconnected_pending_shares: HashMap<Membership::NetworkId, VecDeque<DaShare>>,
disconnected_pending_shares:
HashMap<Membership::NetworkId, VecDeque<dispersal::DispersalRequest>>,
/// Already connected peers connection Ids
connected_peers: HashMap<PeerId, ConnectionId>,
/// List of peers that already has pending open stream request.
@@ -200,7 +203,11 @@ where
pending_shares_sender: UnboundedSender<(Membership::NetworkId, DaShare)>,
/// Pending blobs stream
pending_shares_stream: BoxStream<'static, (Membership::NetworkId, DaShare)>,
/// Waker for dispersal polling
/// Dispersal hook of pending tx channel
pending_tx_sender: UnboundedSender<(Membership::NetworkId, SignedMantleTx)>,
/// Pending tx stream
pending_tx_stream: BoxStream<'static, (Membership::NetworkId, SignedMantleTx)>,
/// Waker for dispersal pollin
waker: Option<Waker>,
}
@@ -227,6 +234,8 @@ where
let (pending_shares_sender, receiver) = mpsc::unbounded_channel();
let pending_shares_stream = UnboundedReceiverStream::new(receiver).boxed();
let (pending_tx_sender, receiver) = mpsc::unbounded_channel();
let pending_tx_stream = UnboundedReceiverStream::new(receiver).boxed();
let disconnected_pending_shares = HashMap::new();
Self {
@@ -244,6 +253,8 @@ where
pending_out_streams,
pending_shares_sender,
pending_shares_stream,
pending_tx_sender,
pending_tx_stream,
waker: None,
}
}
@@ -270,16 +281,19 @@ where
self.pending_shares_sender.clone()
}
/// Get a hook to the sender channel of the shares dispersal events
pub fn tx_sender(&self) -> UnboundedSender<(Membership::NetworkId, SignedMantleTx)> {
self.pending_tx_sender.clone()
}
/// Task for handling streams, one message at a time
/// Writes the blob to the stream and waits for an acknowledgment response
async fn stream_disperse(
mut stream: DispersalStream,
message: DaShare,
message: dispersal::DispersalRequest,
subnetwork_id: SubnetworkId,
) -> Result<StreamHandlerFutureSuccess, DispersalError> {
let blob_id = message.blob_id();
let blob_id: BlobId = blob_id.clone().try_into().unwrap();
let message = dispersal::DispersalRequest::new(Share::new(blob_id, message), subnetwork_id);
let blob_id = message.blob_id().ok_or(DispersalError::NoBlobId)?;
let peer_id = stream.peer_id;
pack_to_writer(&message, &mut stream.stream)
.map_err(|error| DispersalError::Io {
@@ -315,7 +329,7 @@ where
/// it will get scheduled to run otherwise it is parked as idle.
fn handle_stream(
tasks: &FuturesUnordered<StreamHandlerFuture>,
to_disperse: &mut HashMap<PeerId, VecDeque<(SubnetworkId, DaShare)>>,
to_disperse: &mut HashMap<PeerId, VecDeque<(SubnetworkId, dispersal::DispersalRequest)>>,
idle_streams: &mut HashMap<PeerId, DispersalStream>,
stream: DispersalStream,
cx: &Context<'_>,
@@ -335,8 +349,8 @@ where
/// Get a pending request if its available
fn next_request(
peer_id: &PeerId,
to_disperse: &mut HashMap<PeerId, VecDeque<(SubnetworkId, DaShare)>>,
) -> Option<(SubnetworkId, DaShare)> {
to_disperse: &mut HashMap<PeerId, VecDeque<(SubnetworkId, dispersal::DispersalRequest)>>,
) -> Option<(SubnetworkId, dispersal::DispersalRequest)> {
to_disperse.get_mut(peer_id).and_then(VecDeque::pop_front)
}
@@ -354,33 +368,29 @@ where
{
/// Schedule a new task for sending the blob, if stream is not available
/// queue messages for later processing.
fn disperse_share(
tasks: &FuturesUnordered<StreamHandlerFuture>,
idle_streams: &mut HashMap<Membership::Id, DispersalStream>,
membership: &Membership,
connected_peers: &HashMap<PeerId, ConnectionId>,
to_disperse: &mut HashMap<PeerId, VecDeque<(Membership::NetworkId, DaShare)>>,
fn disperse_request(
&mut self,
subnetwork_id: SubnetworkId,
share: &DaShare,
request: &dispersal::DispersalRequest,
) {
let members = membership.members_of(&subnetwork_id);
let members = self.membership.members_of(&subnetwork_id);
let peers = members
.iter()
.filter(|peer_id| connected_peers.contains_key(peer_id));
.filter(|peer_id| self.connected_peers.contains_key(peer_id));
// We may be connected to more than a single node. Usually will be one, but that
// is an internal decision of the executor itself.
for peer in peers {
if let Some(stream) = idle_streams.remove(peer) {
if let Some(stream) = self.idle_streams.remove(peer) {
// push a task if the stream is immediately available
let fut = Self::stream_disperse(stream, share.clone(), subnetwork_id).boxed();
tasks.push(fut);
let fut = Self::stream_disperse(stream, request.clone(), subnetwork_id).boxed();
self.tasks.push(fut);
} else {
// otherwise queue the blob
to_disperse
self.to_disperse
.entry(*peer)
.or_default()
.push_back((subnetwork_id, share.clone()));
.push_back((subnetwork_id, request.clone()));
}
}
}
@@ -388,8 +398,11 @@ where
fn reschedule_shares_for_peer_stream(
stream: &DispersalStream,
membership: &Membership,
to_disperse: &mut HashMap<PeerId, VecDeque<(SubnetworkId, DaShare)>>,
disconnected_pending_shares: &mut HashMap<SubnetworkId, VecDeque<DaShare>>,
to_disperse: &mut HashMap<PeerId, VecDeque<(SubnetworkId, dispersal::DispersalRequest)>>,
disconnected_pending_shares: &mut HashMap<
SubnetworkId,
VecDeque<dispersal::DispersalRequest>,
>,
) {
let peer_id = stream.peer_id;
let subnetworks = membership.membership(&peer_id);
@@ -401,41 +414,38 @@ where
}
}
fn try_open_stream(
pending_out_streams_sender: &UnboundedSender<PeerId>,
membership: &Membership,
connected_peers: &HashMap<PeerId, ConnectionId>,
pending_peer_open_stream_requests: &mut HashSet<PeerId>,
subnetwork_id: SubnetworkId,
) {
let members = membership.members_of(&subnetwork_id);
fn try_open_stream(&mut self, subnetwork_id: SubnetworkId) {
let members = self.membership.members_of(&subnetwork_id);
let peers: Vec<_> = members
.iter()
.filter(|peer_id| connected_peers.contains_key(peer_id))
.filter(|peer_id| !pending_peer_open_stream_requests.contains(peer_id))
.filter(|peer_id| self.connected_peers.contains_key(peer_id))
.filter(|peer_id| !self.pending_peer_open_stream_requests.contains(peer_id))
.collect();
for peer in peers {
if let Err(e) = pending_out_streams_sender.send(*peer) {
if let Err(e) = self.pending_out_streams_sender.send(*peer) {
error!("Error requesting stream for peer {peer}: {e}");
} else {
pending_peer_open_stream_requests.insert(*peer);
self.pending_peer_open_stream_requests.insert(*peer);
}
}
}
fn prune_shares_for_peer(&mut self, peer_id: PeerId) -> VecDeque<(SubnetworkId, DaShare)> {
fn prune_shares_for_peer(
&mut self,
peer_id: PeerId,
) -> VecDeque<(SubnetworkId, dispersal::DispersalRequest)> {
self.to_disperse.remove(&peer_id).unwrap_or_default()
}
fn recover_shares_for_disconnected_subnetworks(&mut self, peer_id: PeerId) {
// push missing blobs into pending ones
let disconnected_pending_shares = self.prune_shares_for_peer(peer_id);
for (subnetwork_id, share) in disconnected_pending_shares {
for (subnetwork_id, req) in disconnected_pending_shares {
self.disconnected_pending_shares
.entry(subnetwork_id)
.or_default()
.push_back(share);
.push_back(req);
}
}
@@ -485,27 +495,28 @@ where
self.pending_shares_stream.poll_next_unpin(cx)
{
if self.subnetwork_open_streams.contains(&subnetwork_id) {
Self::disperse_share(
&self.tasks,
&mut self.idle_streams,
&self.membership,
&self.connected_peers,
&mut self.to_disperse,
subnetwork_id,
&share,
);
self.disperse_request(subnetwork_id, &dispersal::DispersalRequest::from(share));
} else {
Self::try_open_stream(
&self.pending_out_streams_sender,
&self.membership,
&self.connected_peers,
&mut self.pending_peer_open_stream_requests,
subnetwork_id,
);
self.try_open_stream(subnetwork_id);
self.disconnected_pending_shares
.entry(subnetwork_id)
.or_default()
.push_back(share);
.push_back(dispersal::DispersalRequest::from(share));
}
cx.waker().wake_by_ref();
}
}
fn poll_pending_txs(&mut self, cx: &mut Context<'_>) {
if let Poll::Ready(Some((subnetwork_id, tx))) = self.pending_tx_stream.poll_next_unpin(cx) {
if self.subnetwork_open_streams.contains(&subnetwork_id) {
self.disperse_request(subnetwork_id, &dispersal::DispersalRequest::from(tx));
} else {
self.try_open_stream(subnetwork_id);
self.disconnected_pending_shares
.entry(subnetwork_id)
.or_default()
.push_back(dispersal::DispersalRequest::from(tx));
}
cx.waker().wake_by_ref();
}
@@ -635,6 +646,8 @@ where
&mut self,
cx: &mut Context<'_>,
) -> Poll<ToSwarm<Self::ToSwarm, THandlerInEvent<Self>>> {
self.waker = Some(cx.waker().clone());
// Poll tasks generated by streams and share requests.
if let Some(event) = self.poll_pending_tasks(cx) {
return Poll::Ready(ToSwarm::GenerateEvent(event));
@@ -643,6 +656,9 @@ where
// Poll and process any pending shares for dispersal.
self.poll_pending_shares(cx);
// Poll and process any pending transactions for dispersal.
self.poll_pending_txs(cx);
// Poll for newly opened outbound streams.
if let Some(event) = self.poll_pending_streams(cx) {
return Poll::Ready(ToSwarm::GenerateEvent(event));
@@ -653,7 +669,6 @@ where
return Poll::Ready(event);
}
self.waker = Some(cx.waker().clone());
Poll::Pending
}
}
nomos-da/network/core/src/protocols/dispersal/mod.rs
+3 -3
View File
@@ -40,7 +40,7 @@ pub mod test {
let mut validator = Swarm::new_ephemeral_tokio(|k2| {
let p2 = PeerId::from_public_key(&k2.public());
neighbours.add_neighbour(p2);
DispersalValidatorBehaviour::new(neighbours.clone())
DispersalValidatorBehaviour::new(p2, neighbours.clone())
});
validator.listen().with_memory_addr_external().await;
@@ -58,8 +58,8 @@ pub mod test {
let mut res = vec![];
loop {
match validator.select_next_some().await {
SwarmEvent::Behaviour(DispersalEvent::IncomingMessage { message }) => {
res.push(message);
SwarmEvent::Behaviour(DispersalEvent::IncomingShare(share)) => {
res.push(share);
}
event => {
info!("Validator event: {event:?}");
nomos-da/network/core/src/protocols/dispersal/validator/behaviour.rs
+74 -24
View File
@@ -14,7 +14,9 @@ use libp2p::{
};
use libp2p_stream::IncomingStreams;
use log::debug;
use nomos_core::mantle::{ops::channel::blob::BlobOp, Op, SignedMantleTx};
use nomos_da_messages::{
common::Share,
dispersal,
packing::{pack_to_writer, unpack_from_reader},
};
@@ -52,12 +54,12 @@ impl Clone for DispersalError {
}
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub enum DispersalEvent {
/// Received a n
IncomingMessage {
message: Box<dispersal::DispersalRequest>,
},
/// Received a network message.
IncomingShare(Box<Share>),
/// Number of currently assigned subnetworks to the node and TX for a blob.
IncomingTx((u16, Box<SignedMantleTx>)),
/// Something went wrong receiving the blob
DispersalError { error: DispersalError },
}
@@ -66,16 +68,29 @@ impl DispersalEvent {
#[must_use]
pub fn share_size(&self) -> Option<usize> {
match self {
Self::IncomingMessage { message } => Some(message.share.data.column_len()),
Self::DispersalError { .. } => None,
Self::IncomingShare(share) => Some(share.data.column_len()),
Self::IncomingTx { .. } | Self::DispersalError { .. } => None,
}
}
}
impl From<Share> for DispersalEvent {
fn from(share: Share) -> Self {
Self::IncomingShare(Box::new(share))
}
}
impl From<(u16, SignedMantleTx)> for DispersalEvent {
fn from(tx: (u16, SignedMantleTx)) -> Self {
Self::IncomingTx((tx.0, Box::new(tx.1)))
}
}
type DispersalTask =
BoxFuture<'static, Result<(PeerId, dispersal::DispersalRequest, Stream), DispersalError>>;
pub struct DispersalValidatorBehaviour<Membership> {
local_peer_id: PeerId,
stream_behaviour: libp2p_stream::Behaviour,
incoming_streams: IncomingStreams,
tasks: FuturesUnordered<DispersalTask>,
@@ -83,7 +98,7 @@ pub struct DispersalValidatorBehaviour<Membership> {
}
impl<Membership: MembershipHandler> DispersalValidatorBehaviour<Membership> {
pub fn new(membership: Membership) -> Self {
pub fn new(local_peer_id: PeerId, membership: Membership) -> Self {
let stream_behaviour = libp2p_stream::Behaviour::new();
let mut stream_control = stream_behaviour.new_control();
let incoming_streams = stream_control
@@ -91,6 +106,7 @@ impl<Membership: MembershipHandler> DispersalValidatorBehaviour<Membership> {
.expect("Just a single accept to protocol is valid");
let tasks = FuturesUnordered::new();
Self {
local_peer_id,
stream_behaviour,
incoming_streams,
tasks,
@@ -98,6 +114,30 @@ impl<Membership: MembershipHandler> DispersalValidatorBehaviour<Membership> {
}
}
fn process_dispersal_request(
request: &dispersal::DispersalRequest,
) -> Option<dispersal::DispersalResponse> {
match request {
dispersal::DispersalRequest::Share(share_request) => {
let blob_id = share_request.share.blob_id;
Some(dispersal::DispersalResponse::BlobId(blob_id))
}
dispersal::DispersalRequest::Tx(signed_mantle_tx) => {
if let Some(Op::ChannelBlob(BlobOp { blob, .. })) =
signed_mantle_tx.mantle_tx.ops.first()
{
Some(dispersal::DispersalResponse::Tx(*blob))
} else {
// Validator does not acknowledge malformed Tx at network level, but validation
// happens at the service level.
// If transaction is invalid in term of ledger, responsible services might
// terminate the connection to this peer.
None
}
}
}
}
/// Stream handling messages task.
/// This task handles a single message receive. Then it writes up the
/// acknowledgment into the same stream as response and finish.
@@ -105,23 +145,21 @@ impl<Membership: MembershipHandler> DispersalValidatorBehaviour<Membership> {
peer_id: PeerId,
mut stream: Stream,
) -> Result<(PeerId, dispersal::DispersalRequest, Stream), DispersalError> {
let message: dispersal::DispersalRequest = unpack_from_reader(&mut stream)
let request: dispersal::DispersalRequest = unpack_from_reader(&mut stream)
.await
.map_err(|error| DispersalError::Io { peer_id, error })?;
let blob_id = message.share.blob_id;
let response = dispersal::DispersalResponse::BlobId(blob_id);
if let Some(response) = Self::process_dispersal_request(&request) {
pack_to_writer(&response, &mut stream)
.await
.map_err(|error| DispersalError::Io { peer_id, error })?;
pack_to_writer(&response, &mut stream)
.await
.map_err(|error| DispersalError::Io { peer_id, error })?;
stream
.flush()
.await
.map_err(|error| DispersalError::Io { peer_id, error })?;
Ok((peer_id, message, stream))
stream
.flush()
.await
.map_err(|error| DispersalError::Io { peer_id, error })?;
}
Ok((peer_id, request, stream))
}
}
@@ -196,6 +234,7 @@ impl<M: MembershipHandler<Id = PeerId, NetworkId = SubnetworkId> + 'static> Netw
cx: &mut Context<'_>,
) -> Poll<ToSwarm<Self::ToSwarm, THandlerInEvent<Self>>> {
let Self {
local_peer_id,
incoming_streams,
tasks,
..
@@ -204,9 +243,20 @@ impl<M: MembershipHandler<Id = PeerId, NetworkId = SubnetworkId> + 'static> Netw
Poll::Ready(Some(Ok((peer_id, message, stream)))) => {
tasks.push(Self::handle_new_stream(peer_id, stream).boxed());
cx.waker().wake_by_ref();
return Poll::Ready(ToSwarm::GenerateEvent(DispersalEvent::IncomingMessage {
message: Box::new(message),
}));
return match message {
dispersal::DispersalRequest::Share(share_request) => {
Poll::Ready(ToSwarm::GenerateEvent(DispersalEvent::IncomingShare(
Box::new(share_request.share),
)))
}
dispersal::DispersalRequest::Tx(signed_mantle_tx) => {
let assignations = self.membership.membership(local_peer_id).len();
Poll::Ready(ToSwarm::GenerateEvent(DispersalEvent::IncomingTx((
assignations as u16,
Box::new(signed_mantle_tx),
))))
}
};
}
Poll::Ready(Some(Err(error))) => {
debug!("Error on dispersal stream {error:?}");
nomos-da/network/core/src/protocols/replication/behaviour.rs
+33 -16
View File
@@ -110,7 +110,12 @@ impl ReplicationEvent {
#[must_use]
pub fn share_size(&self) -> Option<usize> {
match self {
Self::IncomingMessage { message, .. } => Some(message.share.data.column_len()),
Self::IncomingMessage { message, .. } => match message.as_ref() {
ReplicationRequest::Share(share_request) => {
Some(share_request.share.data.column_len())
}
ReplicationRequest::Tx(_) => None,
},
Self::ReplicationError { .. } => None,
}
}
@@ -318,20 +323,32 @@ where
}
self.seen_message_cache.cache_set(message_id, ());
// Push a message in the queue for every single peer connected that is a member
// of the selected subnetwork_id
let peers = self.no_loopback_member_peers_of(message.subnetwork_id);
// At least one message was enqueued
let mut queued = false;
self.connected
.iter()
.filter(|peer_id| peers.contains(peer_id))
.for_each(|peer_id| {
self.pending_outbound
.enqueue_message(*peer_id, message.clone());
queued = true;
});
match &message {
ReplicationRequest::Share(share_request) => {
// Push a message in the queue for every single peer connected that is a member
// of the selected subnetwork_id
let peers = self.no_loopback_member_peers_of(share_request.subnetwork_id);
self.connected
.iter()
.filter(|peer_id| peers.contains(peer_id))
.for_each(|peer_id| {
self.pending_outbound
.enqueue_message(*peer_id, message.clone());
queued = true;
});
}
ReplicationRequest::Tx(_signed_mantle_tx) => {
// Push Tx to all connected peers
self.connected.iter().for_each(|peer_id| {
self.pending_outbound
.enqueue_message(*peer_id, message.clone());
queued = true;
});
}
}
if queued {
waker.map(Waker::wake_by_ref);
@@ -616,6 +633,10 @@ where
&mut self,
cx: &mut Context<'_>,
) -> Poll<ToSwarm<Self::ToSwarm, THandlerInEvent<Self>>> {
// Keep the most recent waker in case `send_message()` is called from
// outside the behaviour
self.waker = Some(cx.waker().clone());
// The incoming message to be returned to the swarm **after** all the polling is
// done, this way we don't starve the tasks that are polled later in the
// sequence
@@ -650,10 +671,6 @@ where
return incoming_message;
}
// Keep the most recent waker in case `send_message()` is called from
// outside the behaviour
self.waker = Some(cx.waker().clone());
Poll::Pending
}
}
nomos-da/network/core/src/protocols/replication/fixtures/messages.bincode
BIN
View File
Binary file not shown.
nomos-da/network/core/src/protocols/replication/mod.rs
+139 -14
View File
@@ -2,14 +2,39 @@ pub mod behaviour;
#[cfg(test)]
mod test {
use std::{collections::VecDeque, ops::Range, path::PathBuf, sync::LazyLock, time::Duration};
use std::{
collections::{HashSet, VecDeque},
ops::Range,
path::PathBuf,
sync::LazyLock,
time::Duration,
};
use futures::StreamExt as _;
use kzgrs_backend::testutils;
use libp2p::{identity::Keypair, quic, swarm::SwarmEvent, Multiaddr, PeerId, Swarm};
use libp2p::{
identity::{Keypair, PublicKey},
quic,
swarm::SwarmEvent,
Multiaddr, PeerId, Swarm,
};
use libp2p_swarm_test::SwarmExt as _;
use log::info;
use nomos_da_messages::{common::Share, replication::ReplicationRequest};
use nomos_core::{
mantle::{
ledger::Tx as LedgerTx,
ops::{
channel::{blob::BlobOp, ChannelId, Ed25519PublicKey, MsgId},
Op,
},
MantleTx, SignedMantleTx, Transaction as _,
},
proofs::zksig::{DummyZkSignature, ZkSignaturePublic},
};
use nomos_da_messages::{
common::Share,
replication::{ReplicationRequest, ReplicationResponseId},
};
use tokio::sync::mpsc;
use tracing_subscriber::{fmt::TestWriter, EnvFilter};
@@ -77,7 +102,7 @@ mod test {
message, ..
}) = swarm.select_next_some().await
{
if &message == expected_message {
if *message == *expected_message.as_ref() {
break;
}
}
@@ -103,17 +128,14 @@ mod test {
let messages = (0..20u8)
.map(|i| {
blob_id[31] = i;
ReplicationRequest {
share: Share {
blob_id,
data: {
let mut data = testutils::get_default_da_blob_data();
*data.last_mut().unwrap() = i;
testutils::get_da_share(Some(data))
},
ReplicationRequest::from(Share {
blob_id,
data: {
let mut data = testutils::get_default_da_blob_data();
*data.last_mut().unwrap() = i;
testutils::get_da_share(Some(data))
},
subnetwork_id: 0,
}
})
})
.collect::<Vec<_>>();
let serialized = bincode::serialize(&messages).unwrap();
@@ -280,4 +302,107 @@ mod test {
}
}
}
fn get_ed25519_bytes(pubkey: PublicKey) -> Option<[u8; 32]> {
pubkey.try_into_ed25519().ok().map(|pk| pk.to_bytes())
}
#[tokio::test]
async fn test_tx_replication() {
const TX_COUNT: u64 = 5;
let k1 = Keypair::generate_ed25519();
let k2 = Keypair::generate_ed25519();
let peer_id2 = PeerId::from_public_key(&k2.public());
let signer_bytes_k2 = get_ed25519_bytes(k1.public()).expect("Public key must be Ed25519");
let neighbours = make_neighbours(&[&k1, &k2]);
let mut swarm1 = get_swarm(k1, neighbours.clone());
let mut swarm2 = get_swarm(k2, neighbours);
let base_op = Op::ChannelBlob(BlobOp {
channel: ChannelId::from([2; 32]),
blob: [0u8; 32],
blob_size: 0,
da_storage_gas_price: 0,
parent: MsgId::root(),
signer: Ed25519PublicKey::from_bytes(&signer_bytes_k2).unwrap(),
});
let base_mantle_tx = MantleTx {
ops: vec![base_op],
ledger_tx: LedgerTx::new(vec![], vec![]),
storage_gas_price: 0,
execution_gas_price: 0,
};
let base_signed_tx = SignedMantleTx {
ops_profs: Vec::new(),
ledger_tx_proof: DummyZkSignature::prove(ZkSignaturePublic {
msg_hash: base_mantle_tx.hash().into(),
pks: vec![],
}),
mantle_tx: base_mantle_tx,
};
let addr1: Multiaddr = "/ip4/127.0.0.1/udp/5056/quic-v1".parse().unwrap();
swarm1.listen_on(addr1.clone()).unwrap();
let task1 = async move {
swarm2.dial_and_wait(addr1).await;
for i in 0..TX_COUNT {
let mut unique_signed_tx = base_signed_tx.clone();
// Mantle op payload is not yet included in the signed bytes for tx hash, but
// storage_gas_price also affect the hash and is enough in this test case.
unique_signed_tx.mantle_tx.storage_gas_price = i;
let tx_message = ReplicationRequest::from(unique_signed_tx.clone());
// Send each message two times.
swarm2.behaviour_mut().send_message(&tx_message);
swarm2.behaviour_mut().send_message(&tx_message);
}
swarm2.loop_on_next().await;
};
tokio::spawn(task1);
wait_for_incoming_connection(&mut swarm1, peer_id2).await;
let mut received_tx_ids: HashSet<ReplicationResponseId> = HashSet::new();
let mut duplicate_messages_count = 0;
for _ in 0..TX_COUNT {
let event = tokio::time::timeout(
Duration::from_secs(5),
swarm1.wait(|event| {
if let SwarmEvent::Behaviour(ReplicationEvent::IncomingMessage {
message,
..
}) = event
{
if let ReplicationRequest::Tx(_) = message.as_ref() {
return Some(message);
}
}
None
}),
)
.await
.expect("Swarm1 should receive all Tx messages within timeout");
let received_tx_id = event.id();
if !received_tx_ids.insert(received_tx_id) {
duplicate_messages_count += 1;
}
}
assert_eq!(received_tx_ids.len(), TX_COUNT as usize, "Txs not received");
assert_eq!(
duplicate_messages_count, 0,
"No duplicate Tx messages should be replicated."
);
}
}
nomos-da/network/core/src/swarm/common/handlers.rs
+54 -15
View File
@@ -1,7 +1,7 @@
use kzgrs_backend::common::share::DaShare;
use futures::channel::oneshot;
use libp2p::PeerId;
use log::{debug, error};
use nomos_da_messages::replication;
use nomos_da_messages::replication::ReplicationRequest;
use subnetworks_assignations::MembershipHandler;
use tokio::sync::mpsc::UnboundedSender;
@@ -12,26 +12,47 @@ use crate::{
replication::behaviour::{ReplicationBehaviour, ReplicationEvent},
sampling::SamplingEvent,
},
swarm::{DispersalValidationError, DispersalValidatorEvent},
SubnetworkId,
};
pub async fn handle_validator_dispersal_event<Membership>(
validation_events_sender: &UnboundedSender<DaShare>,
validation_events_sender: &UnboundedSender<DispersalValidatorEvent>,
replication_behaviour: &mut ReplicationBehaviour<Membership>,
event: DispersalEvent,
) where
Membership: MembershipHandler<NetworkId = SubnetworkId, Id = PeerId>,
{
let (sender, receiver) = oneshot::channel();
let validation_event = DispersalValidatorEvent {
event: event.clone(),
sender: Some(sender),
};
if let Err(e) = validation_events_sender.send(validation_event) {
error!("Error sending blob to validation: {e:?}");
}
let Ok(validation_result) = receiver.await else {
error!("Error receiving dispersal validation result");
return;
};
// Do not replicate if validation fails.
if matches!(validation_result, Err(DispersalValidationError)) {
error!("Error validating dispersal event: {event:?}");
return;
}
// Send message for replication
if let DispersalEvent::IncomingMessage { message } = event {
let share_message = message.share;
if let Err(e) = validation_events_sender.send(share_message.data.clone()) {
error!("Error sending blob to validation: {e:?}");
match event {
DispersalEvent::IncomingShare(share) => {
replication_behaviour.send_message(&ReplicationRequest::from(*share));
}
replication_behaviour.send_message(&replication::ReplicationRequest::new(
share_message,
message.subnetwork_id,
));
DispersalEvent::IncomingTx(signed_mantle_tx) => {
replication_behaviour.send_message(&ReplicationRequest::from(*signed_mantle_tx.1));
}
DispersalEvent::DispersalError { .. } => {} // Do not replicate errors.
}
}
@@ -44,12 +65,30 @@ pub async fn handle_sampling_event(
}
}
pub async fn handle_replication_event(
validation_events_sender: &UnboundedSender<DaShare>,
pub async fn handle_replication_event<Membership>(
validation_events_sender: &UnboundedSender<DispersalValidatorEvent>,
membership: &Membership,
peer_id: &<Membership as MembershipHandler>::Id,
event: ReplicationEvent,
) {
) where
Membership: MembershipHandler + Send + Sync,
<Membership as MembershipHandler>::Id: Send + Sync,
{
if let ReplicationEvent::IncomingMessage { message, .. } = event {
if let Err(e) = validation_events_sender.send(message.share.data) {
let dispersal_event = match *message {
ReplicationRequest::Share(share_request) => DispersalEvent::from(share_request.share),
ReplicationRequest::Tx(tx) => {
let assignations = membership.membership(peer_id).len();
DispersalEvent::from((assignations as u16, tx))
}
};
let validation_event = DispersalValidatorEvent {
event: dispersal_event,
sender: None,
};
if let Err(e) = validation_events_sender.send(validation_event) {
error!("Error sending blob to validation: {e:?}");
}
}
nomos-da/network/core/src/swarm/common/monitor.rs
+2 -1
View File
@@ -65,7 +65,8 @@ impl From<&DispersalExecutorEvent> for MonitorEvent {
impl From<&DispersalValidatorEvent> for MonitorEvent {
fn from(event: &DispersalValidatorEvent) -> Self {
match event {
DispersalValidatorEvent::IncomingMessage { .. } => Self::Noop,
DispersalValidatorEvent::IncomingShare { .. }
| DispersalValidatorEvent::IncomingTx { .. } => Self::Noop,
DispersalValidatorEvent::DispersalError { error } => {
Self::ValidatorDispersal(error.clone())
}
nomos-da/network/core/src/swarm/executor.rs
+23 -5
View File
@@ -9,7 +9,7 @@ use libp2p::{
Multiaddr, PeerId, Swarm, SwarmBuilder, TransportError,
};
use log::debug;
use nomos_core::da::BlobId;
use nomos_core::{da::BlobId, mantle::SignedMantleTx};
use subnetworks_assignations::MembershipHandler;
use tokio::{
sync::mpsc::{unbounded_channel, UnboundedSender},
@@ -38,7 +38,8 @@ use crate::{
policy::DAConnectionPolicy,
},
validator::{SampleArgs, SwarmSettings, ValidatorEventsStream},
BalancerStats, ConnectionBalancer, ConnectionMonitor, MonitorStats,
BalancerStats, ConnectionBalancer, ConnectionMonitor, DispersalValidatorEvent,
MonitorStats,
},
SubnetworkId,
};
@@ -69,8 +70,9 @@ where
>,
>,
sampling_events_sender: UnboundedSender<SamplingEvent>,
validation_events_sender: UnboundedSender<DaShare>,
validation_events_sender: UnboundedSender<DispersalValidatorEvent>,
dispersal_events_sender: UnboundedSender<DispersalExecutorEvent>,
membership: Membership,
phantom: PhantomData<HistoricMembership>,
}
@@ -124,7 +126,7 @@ where
Self {
swarm: Self::build_swarm(
key,
membership,
membership.clone(),
addressbook,
balancer,
monitor,
@@ -136,6 +138,7 @@ where
sampling_events_sender,
validation_events_sender,
dispersal_events_sender,
membership,
phantom: PhantomData,
},
ExecutorEventsStream {
@@ -231,6 +234,15 @@ where
.shares_sender()
}
pub fn dispersal_tx_channel(
&mut self,
) -> UnboundedSender<(Membership::NetworkId, SignedMantleTx)> {
self.swarm
.behaviour()
.dispersal_executor_behaviour()
.tx_sender()
}
pub fn dispersal_open_stream_sender(&mut self) -> UnboundedSender<PeerId> {
self.swarm
.behaviour()
@@ -319,7 +331,13 @@ where
self.swarm.behaviour_mut().monitor_behaviour_mut(),
MonitorEvent::from(&event),
);
handle_replication_event(&self.validation_events_sender, event).await;
handle_replication_event(
&self.validation_events_sender,
&self.membership,
self.local_peer_id(),
event,
)
.await;
}
#[expect(
nomos-da/network/core/src/swarm/mod.rs
+18 -2
View File
@@ -3,8 +3,14 @@ pub mod executor;
pub mod validator;
pub use common::{
monitor::DAConnectionMonitorSettings, policy::DAConnectionPolicySettings, ReplicationConfig,
balancer::BalancerStats,
monitor::{dto::MonitorStats, DAConnectionMonitorSettings},
policy::DAConnectionPolicySettings,
ReplicationConfig,
};
use futures::channel::oneshot;
use crate::protocols::dispersal::validator::behaviour::DispersalEvent;
pub(crate) type ConnectionMonitor<Membership> =
common::monitor::DAConnectionMonitor<common::policy::DAConnectionPolicy<Membership>>;
@@ -14,4 +20,14 @@ pub(crate) type ConnectionBalancer<Membership> = common::balancer::DAConnectionB
common::policy::DAConnectionPolicy<Membership>,
>;
pub use common::{balancer::BalancerStats, monitor::dto::MonitorStats};
/// Dispersed data failed to be validated.
pub struct DispersalValidationError;
pub type DispersalValidationResult = Result<(), DispersalValidationError>;
pub type ValidationResultSender = Option<oneshot::Sender<DispersalValidationResult>>;
pub struct DispersalValidatorEvent {
pub event: DispersalEvent,
pub sender: ValidationResultSender,
}
nomos-da/network/core/src/swarm/validator.rs
+13 -5
View File
@@ -1,7 +1,6 @@
use std::{collections::HashSet, io, marker::PhantomData, time::Duration};
use futures::{stream, StreamExt as _};
use kzgrs_backend::common::share::DaShare;
use libp2p::{
core::transport::ListenerId,
identity::Keypair,
@@ -17,6 +16,7 @@ use tokio::{
};
use tokio_stream::wrappers::{IntervalStream, UnboundedReceiverStream};
use super::DispersalValidatorEvent;
use crate::{
addressbook::AddressBookHandler,
behaviour::validator::{ValidatorBehaviour, ValidatorBehaviourEvent},
@@ -59,7 +59,7 @@ pub struct SwarmSettings {
pub struct ValidatorEventsStream {
pub sampling_events_receiver: UnboundedReceiverStream<SamplingEvent>,
pub validation_events_receiver: UnboundedReceiverStream<DaShare>,
pub validation_events_receiver: UnboundedReceiverStream<DispersalValidatorEvent>,
}
pub struct ValidatorSwarm<Membership, HistoricMembership, Addressbook>
@@ -78,7 +78,8 @@ where
>,
>,
sampling_events_sender: UnboundedSender<SamplingEvent>,
validation_events_sender: UnboundedSender<DaShare>,
validation_events_sender: UnboundedSender<DispersalValidatorEvent>,
membership: Membership,
phantom: PhantomData<HistoricMembership>,
}
@@ -132,7 +133,7 @@ where
Self {
swarm: Self::build_swarm(
key,
membership,
membership.clone(),
addressbook,
balancer,
monitor,
@@ -143,6 +144,7 @@ where
),
sampling_events_sender,
validation_events_sender,
membership,
phantom: PhantomData,
},
ValidatorEventsStream {
@@ -297,7 +299,13 @@ where
self.swarm.behaviour_mut().monitor_behaviour_mut(),
MonitorEvent::from(&event),
);
handle_replication_event(&self.validation_events_sender, event).await;
handle_replication_event(
&self.validation_events_sender,
&self.membership,
self.local_peer_id(),
event,
)
.await;
}
#[expect(
nomos-da/network/messages/src/common.rs
+9
View File
@@ -2,6 +2,8 @@ use kzgrs_backend::common::share::{DaLightShare, DaShare};
use nomos_core::da::BlobId;
use serde::{Deserialize, Serialize};
use crate::SubnetworkId;
#[repr(C)]
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct Share {
@@ -30,6 +32,13 @@ impl LightShare {
}
}
#[repr(C)]
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct ShareRequest {
pub share: Share,
pub subnetwork_id: SubnetworkId,
}
#[repr(u8)]
#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)]
pub enum CloseMessageReason {
nomos-da/network/messages/src/dispersal.rs
+38 -11
View File
@@ -1,13 +1,18 @@
use nomos_core::da::BlobId;
use kzgrs_backend::common::share::DaShare;
use nomos_core::{
da::{blob, BlobId},
mantle::{ops::Op, SignedMantleTx},
};
use serde::{Deserialize, Serialize};
use crate::{common::Share, SubnetworkId};
use crate::common::{Share, ShareRequest};
#[repr(C)]
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub enum DispersalErrorType {
ChunkSize,
Verification,
BlobVerification,
TxVerification,
}
#[repr(C)]
@@ -33,25 +38,47 @@ impl DispersalError {
}
#[repr(C)]
#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct DispersalRequest {
pub share: Share,
pub subnetwork_id: SubnetworkId,
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub enum DispersalRequest {
Share(ShareRequest),
Tx(SignedMantleTx),
}
impl DispersalRequest {
#[must_use]
pub const fn new(share: Share, subnetwork_id: SubnetworkId) -> Self {
Self {
share,
subnetwork_id,
pub fn blob_id(&self) -> Option<BlobId> {
match self {
Self::Share(share) => Some(share.share.blob_id),
Self::Tx(tx) => match tx.mantle_tx.ops.first() {
Some(Op::ChannelBlob(blob_op)) => Some(blob_op.blob),
_ => None,
},
}
}
}
impl From<DaShare> for DispersalRequest {
fn from(share: DaShare) -> Self {
Self::Share(ShareRequest {
subnetwork_id: share.share_idx,
share: Share {
blob_id: blob::Share::blob_id(&share),
data: share,
},
})
}
}
impl From<SignedMantleTx> for DispersalRequest {
fn from(tx: SignedMantleTx) -> Self {
Self::Tx(tx)
}
}
#[repr(C)]
#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)]
pub enum DispersalResponse {
Tx(BlobId),
BlobId(BlobId),
Error(DispersalError),
}
nomos-da/network/messages/src/packing.rs
+2 -4
View File
@@ -99,8 +99,7 @@ mod tests {
let blob_id = BlobId::from([0; 32]);
let share = get_da_share(None);
let share = Share::new(blob_id, share);
let subnetwork_id = 0;
let message = DispersalRequest::new(share, subnetwork_id);
let message = DispersalRequest::from(share.data);
let packed_message = pack(&message)?;
let unpacked_message: DispersalRequest = unpack(&packed_message)?;
@@ -114,8 +113,7 @@ mod tests {
let blob_id = BlobId::from([0; 32]);
let data = get_da_share(None);
let share = Share::new(blob_id, data);
let subnetwork_id = 0;
let message = DispersalRequest::new(share, subnetwork_id);
let message = DispersalRequest::from(share.data);
let mut writer = Vec::new();
pack_to_writer(&message, &mut writer).await?;
nomos-da/network/messages/src/replication.rs
+36 -12
View File
@@ -1,27 +1,43 @@
use nomos_core::da::BlobId;
use nomos_core::{
da::BlobId,
mantle::{SignedMantleTx, Transaction as _},
};
use serde::{Deserialize, Serialize};
use crate::{common::Share, SubnetworkId};
use crate::{
common::{Share, ShareRequest},
SubnetworkId,
};
#[repr(C)]
#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
pub struct ReplicationRequest {
pub share: Share,
pub subnetwork_id: SubnetworkId,
pub enum ReplicationRequest {
Share(ShareRequest),
Tx(SignedMantleTx),
}
impl ReplicationRequest {
#[must_use]
pub const fn new(share: Share, subnetwork_id: SubnetworkId) -> Self {
Self {
share,
subnetwork_id,
pub fn id(&self) -> ReplicationResponseId {
match self {
Self::Share(share) => (share.share.blob_id, share.subnetwork_id).into(),
Self::Tx(tx) => tx.into(),
}
}
}
#[must_use]
pub fn id(&self) -> ReplicationResponseId {
(self.share.blob_id, self.subnetwork_id).into()
impl From<SignedMantleTx> for ReplicationRequest {
fn from(tx: SignedMantleTx) -> Self {
Self::Tx(tx)
}
}
impl From<Share> for ReplicationRequest {
fn from(share: Share) -> Self {
Self::Share(ShareRequest {
subnetwork_id: share.data.share_idx,
share,
})
}
}
@@ -36,3 +52,11 @@ impl From<(BlobId, SubnetworkId)> for ReplicationResponseId {
Self(id)
}
}
impl From<&SignedMantleTx> for ReplicationResponseId {
fn from(tx: &SignedMantleTx) -> Self {
let mut id = [0; 34];
id[..32].copy_from_slice(&tx.hash().0);
Self(id)
}
}
nomos-services/api/src/http/cl.rs
+12 -58
View File
@@ -12,31 +12,16 @@ use tokio::sync::oneshot;
use crate::wait_with_timeout;
pub type ClMempoolService<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
> = TxMempoolService<
MempoolNetworkAdapter<Tx, <Tx as Transaction>::Hash, RuntimeServiceId>,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
MockPool<HeaderId, Tx, <Tx as Transaction>::Hash>,
RuntimeServiceId,
>;
pub type ClMempoolService<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId> =
TxMempoolService<
MempoolNetworkAdapter<Tx, <Tx as Transaction>::Hash, RuntimeServiceId>,
SamplingNetworkAdapter,
SamplingStorage,
MockPool<HeaderId, Tx, <Tx as Transaction>::Hash>,
RuntimeServiceId,
>;
pub async fn cl_mempool_metrics<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>(
pub async fn cl_mempool_metrics<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>(
handle: &overwatch::overwatch::handle::OverwatchHandle<RuntimeServiceId>,
) -> Result<MempoolMetrics, super::DynError>
where
@@ -45,24 +30,12 @@ where
Ord + Debug + Send + Sync + Serialize + for<'de> Deserialize<'de> + 'static,
SamplingNetworkAdapter:
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
VerifierNetworkAdapter:
nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
VerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
RuntimeServiceId: Debug
+ Sync
+ Send
+ Display
+ AsServiceId<
ClMempoolService<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>,
>,
+ AsServiceId<ClMempoolService<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>>,
{
let relay = handle.relay().await?;
let (sender, receiver) = oneshot::channel();
@@ -80,14 +53,7 @@ where
.await
}
pub async fn cl_mempool_status<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>(
pub async fn cl_mempool_status<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>(
handle: &overwatch::overwatch::handle::OverwatchHandle<RuntimeServiceId>,
items: Vec<<Tx as Transaction>::Hash>,
) -> Result<Vec<Status<HeaderId>>, super::DynError>
@@ -97,24 +63,12 @@ where
Ord + Debug + Send + Sync + Serialize + for<'de> Deserialize<'de> + 'static,
SamplingNetworkAdapter:
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
VerifierNetworkAdapter:
nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
VerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
RuntimeServiceId: Debug
+ Sync
+ Send
+ Display
+ AsServiceId<
ClMempoolService<
Tx,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>,
>,
+ AsServiceId<ClMempoolService<Tx, SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>>,
{
let relay = handle.relay().await?;
let (sender, receiver) = oneshot::channel();
nomos-services/api/src/http/consensus/cryptarchia.rs
-28
View File
@@ -31,9 +31,6 @@ pub type Cryptarchia<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
const SIZE: usize,
@@ -54,9 +51,6 @@ pub type Cryptarchia<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>;
@@ -83,9 +77,6 @@ pub async fn cryptarchia_info<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
const SIZE: usize,
@@ -103,11 +94,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -121,9 +107,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -146,9 +129,6 @@ pub async fn cryptarchia_headers<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
const SIZE: usize,
@@ -168,11 +148,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -186,9 +161,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
nomos-services/api/src/http/da.rs
+51 -83
View File
@@ -13,12 +13,11 @@ use nomos_core::{
BlobId, DaVerifier as CoreDaVerifier,
},
header::HeaderId,
mantle::{select::FillSize as FillSizeWithTx, Transaction},
mantle::{select::FillSize as FillSizeWithTx, SignedMantleTx, Transaction},
};
use nomos_da_dispersal::{
adapters::{mempool::DaMempoolAdapter, network::DispersalNetworkAdapter},
backend::DispersalBackend,
DaDispersalMsg, DispersalService,
adapters::network::DispersalNetworkAdapter, backend::DispersalBackend, DaDispersalMsg,
DispersalService,
};
use nomos_da_indexer::{
consensus::adapters::cryptarchia::CryptarchiaConsensusAdapter,
@@ -37,8 +36,9 @@ use nomos_da_sampling::{
backend::DaSamplingServiceBackend, storage::adapters::rocksdb::converter::DaStorageConverter,
};
use nomos_da_verifier::{
backend::VerifierBackend, storage::adapters::rocksdb::RocksAdapter as VerifierStorageAdapter,
DaVerifierMsg, DaVerifierService,
backend::VerifierBackend, mempool::DaMempoolAdapter,
storage::adapters::rocksdb::RocksAdapter as VerifierStorageAdapter, DaVerifierMsg,
DaVerifierService,
};
use nomos_libp2p::PeerId;
use nomos_mempool::{
@@ -63,9 +63,6 @@ pub type DaIndexer<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
const SIZE: usize,
@@ -87,9 +84,6 @@ pub type DaIndexer<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>;
@@ -116,29 +110,18 @@ pub type DaVerifier<
VerifierBackend,
StorageSerializer,
DaStorageConverter,
VerifierMempoolAdapter,
RuntimeServiceId,
> = DaVerifierService<
VerifierBackend,
NetworkAdapter,
VerifierStorageAdapter<Blob, StorageSerializer, DaStorageConverter>,
VerifierMempoolAdapter,
RuntimeServiceId,
>;
pub type DaDispersal<
Backend,
NetworkAdapter,
MempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
> = DispersalService<
Backend,
NetworkAdapter,
MempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
>;
pub type DaDispersal<Backend, NetworkAdapter, Membership, RuntimeServiceId> =
DispersalService<Backend, NetworkAdapter, Membership, RuntimeServiceId>;
pub type DaNetwork<
Backend,
@@ -156,28 +139,50 @@ pub type DaNetwork<
RuntimeServiceId,
>;
pub async fn add_share<A, S, N, VB, SS, DaStorageConverter, RuntimeServiceId>(
pub async fn add_share<
DaShare,
VerifierNetwork,
ShareVerifier,
SerdeOp,
DaStorageConverter,
VerifierMempoolAdapter,
RuntimeServiceId,
>(
handle: &OverwatchHandle<RuntimeServiceId>,
share: S,
share: DaShare,
) -> Result<Option<()>, DynError>
where
A: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
S: Share + Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
<S as Share>::BlobId: Clone + Send + Sync + 'static,
<S as Share>::ShareIndex: Clone + Eq + Hash + Send + Sync + 'static,
<S as Share>::LightShare: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
<S as Share>::SharesCommitments: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
N: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
N::Settings: Clone,
VB: VerifierBackend + CoreDaVerifier<DaShare = S>,
<VB as VerifierBackend>::Settings: Clone,
<VB as CoreDaVerifier>::Error: Error,
SS: StorageSerde + Send + Sync + 'static,
DaStorageConverter: DaConverter<RocksBackend<SS>, Share = S> + Send + Sync + 'static,
DaShare: Share + Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
<DaShare as Share>::BlobId: Clone + Send + Sync + 'static,
<DaShare as Share>::ShareIndex: Clone + Eq + Hash + Send + Sync + 'static,
<DaShare as Share>::LightShare: Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
<DaShare as Share>::SharesCommitments:
Serialize + DeserializeOwned + Clone + Send + Sync + 'static,
VerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
VerifierNetwork::Settings: Clone,
ShareVerifier: VerifierBackend + CoreDaVerifier<DaShare = DaShare>,
<ShareVerifier as VerifierBackend>::Settings: Clone,
<ShareVerifier as CoreDaVerifier>::Error: Error,
SerdeOp: StorageSerde + Send + Sync + 'static,
DaStorageConverter: DaConverter<RocksBackend<SerdeOp>, Share = DaShare, Tx = SignedMantleTx>
+ Send
+ Sync
+ 'static,
VerifierMempoolAdapter: DaMempoolAdapter,
RuntimeServiceId: Debug
+ Sync
+ Display
+ AsServiceId<DaVerifier<S, N, VB, SS, DaStorageConverter, RuntimeServiceId>>,
+ AsServiceId<
DaVerifier<
DaShare,
VerifierNetwork,
ShareVerifier,
SerdeOp,
DaStorageConverter,
VerifierMempoolAdapter,
RuntimeServiceId,
>,
>,
{
let relay = handle.relay().await?;
let (sender, receiver) = oneshot::channel();
@@ -200,9 +205,6 @@ pub async fn get_range<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
const SIZE: usize,
@@ -248,11 +250,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -268,9 +265,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
SIZE,
@@ -291,17 +285,9 @@ where
wait_with_timeout(receiver, "Timeout while waiting for get range".to_owned()).await
}
pub async fn disperse_data<
Backend,
NetworkAdapter,
MempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
>(
pub async fn disperse_data<Backend, NetworkAdapter, Membership, RuntimeServiceId>(
handle: &OverwatchHandle<RuntimeServiceId>,
data: Vec<u8>,
metadata: Metadata,
) -> Result<Backend::BlobId, DynError>
where
Membership: MembershipHandler<NetworkId = SubnetworkId, Id = PeerId>
@@ -310,38 +296,20 @@ where
+ Send
+ Sync
+ 'static,
Backend: DispersalBackend<
NetworkAdapter = NetworkAdapter,
MempoolAdapter = MempoolAdapter,
Metadata = Metadata,
> + Send
+ Sync
+ 'static,
Backend: DispersalBackend<NetworkAdapter = NetworkAdapter> + Send + Sync + 'static,
Backend::Settings: Clone + Send + Sync,
Backend::BlobId: Serialize,
NetworkAdapter: DispersalNetworkAdapter<SubnetworkId = Membership::NetworkId> + Send,
MempoolAdapter: DaMempoolAdapter,
Metadata: metadata::Metadata + Debug + Send + 'static,
RuntimeServiceId: Debug
+ Sync
+ Display
+ AsServiceId<
DaDispersal<
Backend,
NetworkAdapter,
MempoolAdapter,
Membership,
Metadata,
RuntimeServiceId,
>,
>,
+ AsServiceId<DaDispersal<Backend, NetworkAdapter, Membership, RuntimeServiceId>>,
{
let relay = handle.relay().await?;
let (sender, receiver) = oneshot::channel();
relay
.send(DaDispersalMsg::Disperse {
data,
metadata,
reply_channel: sender,
})
.await
nomos-services/api/src/http/mempool.rs
-19
View File
@@ -5,7 +5,6 @@ use nomos_core::{da::blob::info::DispersedBlobInfo, header::HeaderId};
use nomos_da_sampling::{
backend::DaSamplingServiceBackend, network::NetworkAdapter as DaSamplingNetworkAdapter,
};
use nomos_da_verifier::backend::VerifierBackend;
use nomos_mempool::{
backend::mockpool::MockPool, network::NetworkAdapter, DaMempoolService, MempoolMsg,
TxMempoolService,
@@ -21,9 +20,7 @@ pub async fn add_tx<
MempoolNetworkBackend,
MempoolNetworkAdapter,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
Item,
Key,
RuntimeServiceId,
@@ -40,10 +37,7 @@ where
+ 'static,
MempoolNetworkAdapter::Settings: Send + Sync,
SamplingNetworkAdapter: DaSamplingNetworkAdapter<RuntimeServiceId> + Send + Sync,
VerifierNetworkAdapter:
nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
VerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
Item: Clone + Debug + Send + Sync + Serialize + for<'de> Deserialize<'de> + 'static,
Key: Clone + Debug + Ord + Hash + Send + Serialize + for<'de> Deserialize<'de> + 'static,
RuntimeServiceId: Debug
@@ -54,9 +48,7 @@ where
TxMempoolService<
MempoolNetworkAdapter,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
MockPool<HeaderId, Item, Key>,
RuntimeServiceId,
>,
@@ -87,9 +79,6 @@ pub async fn add_blob_info<
SamplingBackend,
SamplingAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>(
handle: &overwatch::overwatch::handle::OverwatchHandle<RuntimeServiceId>,
@@ -109,11 +98,6 @@ where
SamplingBackend::Settings: Clone,
SamplingAdapter: DaSamplingNetworkAdapter<RuntimeServiceId> + Send,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierBackend: VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
RuntimeServiceId: Debug
+ Sync
+ Display
@@ -124,9 +108,6 @@ where
SamplingBackend,
SamplingAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>,
nomos-services/chain-service/Cargo.toml
-1
View File
@@ -16,7 +16,6 @@ futures = "0.3"
nomos-blend-service = { workspace = true }
nomos-core = { workspace = true }
nomos-da-sampling = { workspace = true }
nomos-da-verifier = { workspace = true }
nomos-ledger = { workspace = true, features = ["serde"] }
nomos-mempool = { workspace = true }
nomos-network = { workspace = true }
nomos-services/chain-service/src/lib.rs
+4 -64
View File
@@ -254,9 +254,6 @@ pub struct CryptarchiaConsensus<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
> where
@@ -289,11 +286,6 @@ pub struct CryptarchiaConsensus<
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
{
@@ -315,9 +307,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
> ServiceData
@@ -334,9 +323,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>
@@ -369,11 +355,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
{
@@ -409,9 +390,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
> ServiceCore<RuntimeServiceId>
@@ -428,9 +406,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>
@@ -503,11 +478,6 @@ where
SamplingNetworkAdapter:
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + Sync + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -522,9 +492,7 @@ where
TxMempoolService<
ClPoolAdapter,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
ClPool,
RuntimeServiceId,
>,
@@ -536,9 +504,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>
@@ -547,9 +512,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>
@@ -582,9 +544,8 @@ where
SamplingBackend,
Storage,
TxS,
DaVerifierBackend,
RuntimeServiceId,
> = CryptarchiaConsensusRelays::from_service_resources_handle::<_, _, _, _, _>(
> = CryptarchiaConsensusRelays::from_service_resources_handle::<_, _, _>(
&self.service_resources_handle,
)
.await;
@@ -666,9 +627,9 @@ where
Some(Duration::from_secs(60)),
NetworkService<_, _>,
BlendService,
TxMempoolService<_, _, _, _, _, _, _>,
DaMempoolService<_, _, _, _, _, _, _, _, _>,
DaSamplingService<_, _, _, _, _, _, _>,
TxMempoolService<_, _, _, _, _>,
DaMempoolService<_, _, _, _, _, _>,
DaSamplingService<_, _, _, _>,
StorageService<_, _>,
TimeService<_, _>
)
@@ -922,9 +883,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>
@@ -941,9 +899,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>
@@ -1012,11 +967,6 @@ where
SamplingBackend::BlobId: Debug + Ord + Send + Sync + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + Sync + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
{
@@ -1098,7 +1048,6 @@ where
SamplingBackend,
Storage,
TxS,
DaVerifierBackend,
RuntimeServiceId,
>,
block_subscription_sender: &broadcast::Sender<Block<ClPool::Item, DaPool::Item>>,
@@ -1168,7 +1117,6 @@ where
/// Try to add a [`Block`] to [`Cryptarchia`].
/// A [`Block`] is only added if it's valid
#[expect(clippy::allow_attributes_without_reason)]
#[expect(clippy::type_complexity)]
#[instrument(level = "debug", skip(cryptarchia, relays))]
async fn process_block(
cryptarchia: Cryptarchia,
@@ -1184,7 +1132,6 @@ where
SamplingBackend,
Storage,
TxS,
DaVerifierBackend,
RuntimeServiceId,
>,
block_broadcaster: &broadcast::Sender<Block<ClPool::Item, DaPool::Item>>,
@@ -1250,7 +1197,6 @@ where
}
#[expect(clippy::allow_attributes_without_reason)]
#[expect(clippy::type_complexity)]
#[instrument(level = "debug", skip(tx_selector, blob_selector, relays))]
async fn propose_block(
parent: HeaderId,
@@ -1269,7 +1215,6 @@ where
SamplingBackend,
Storage,
TxS,
DaVerifierBackend,
RuntimeServiceId,
>,
) -> Option<Block<ClPool::Item, DaPool::Item>> {
@@ -1402,10 +1347,6 @@ where
/// the consensus.
/// * `block_subscription_sender` - The broadcast channel to send the blocks
/// to the services.
#[expect(
clippy::type_complexity,
reason = "CryptarchiaConsensusState and CryptarchiaConsensusRelays amount of generics."
)]
async fn initialize_cryptarchia(
&self,
genesis_id: HeaderId,
@@ -1423,7 +1364,6 @@ where
SamplingBackend,
Storage,
TxS,
DaVerifierBackend,
RuntimeServiceId,
>,
) -> (Cryptarchia, PrunedBlocks<HeaderId>, Leader) {
nomos-services/chain-service/src/relays.rs
+4 -33
View File
@@ -1,7 +1,6 @@
use std::{
fmt::{Debug, Display},
hash::Hash,
marker::PhantomData,
};
use nomos_core::{
@@ -60,7 +59,6 @@ pub struct CryptarchiaConsensusRelays<
SamplingBackend,
Storage,
TxS,
DaVerifierBackend,
RuntimeServiceId,
> where
BlendService: ServiceData,
@@ -73,7 +71,6 @@ pub struct CryptarchiaConsensusRelays<
Storage: StorageBackend + Send + Sync + 'static,
SamplingBackend: DaSamplingServiceBackend,
TxS: TxSelect,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend,
{
network_relay: NetworkRelay<
<NetworkAdapter as network::NetworkAdapter<RuntimeServiceId>>::Backend,
@@ -86,7 +83,6 @@ pub struct CryptarchiaConsensusRelays<
storage_adapter: StorageAdapter<Storage, TxS::Tx, BS::BlobId, RuntimeServiceId>,
sampling_relay: SamplingRelay<DaPool::Key>,
time_relay: TimeRelay,
_phantom_data: PhantomData<DaVerifierBackend>,
}
impl<
@@ -100,7 +96,6 @@ impl<
SamplingBackend,
Storage,
TxS,
DaVerifierBackend,
RuntimeServiceId,
>
CryptarchiaConsensusRelays<
@@ -114,7 +109,6 @@ impl<
SamplingBackend,
Storage,
TxS,
DaVerifierBackend,
RuntimeServiceId,
>
where
@@ -146,8 +140,6 @@ where
TryFrom<Block<ClPool::Item, DaPool::Item>> + TryInto<Block<ClPool::Item, DaPool::Item>>,
TxS: TxSelect<Tx = ClPool::Item>,
TxS::Settings: Send,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + Sync + 'static,
DaVerifierBackend::Settings: Clone,
{
pub async fn new(
network_relay: NetworkRelay<
@@ -174,10 +166,9 @@ where
blend_relay,
cl_mempool_relay,
da_mempool_relay,
sampling_relay,
storage_adapter,
sampling_relay,
time_relay,
_phantom_data: PhantomData,
}
}
@@ -186,8 +177,6 @@ where
pub async fn from_service_resources_handle<
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
>(
service_resources_handle: &OpaqueServiceResourcesHandle<
@@ -204,9 +193,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>,
@@ -226,13 +212,6 @@ where
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage:
nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierStorage:
nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + Sync + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork:
nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierNetwork::Settings: Clone,
TimeBackend: TimeBackendTrait,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -246,9 +225,7 @@ where
TxMempoolService<
ClPoolAdapter,
SamplingNetworkAdapter,
DaVerifierNetwork,
SamplingStorage,
DaVerifierStorage,
ClPool,
RuntimeServiceId,
>,
@@ -260,9 +237,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>
@@ -271,9 +245,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>
@@ -297,19 +268,19 @@ where
let cl_mempool_relay = service_resources_handle
.overwatch_handle
.relay::<TxMempoolService<_, _, _, _, _, _, _>>()
.relay::<TxMempoolService<_, _, _, _, _>>()
.await
.expect("Relay connection with CL MemPoolService should succeed");
let da_mempool_relay = service_resources_handle
.overwatch_handle
.relay::<DaMempoolService<_, _, _, _, _, _, _, _, _>>()
.relay::<DaMempoolService<_, _, _, _, _, _>>()
.await
.expect("Relay connection with DA MemPoolService should succeed");
let sampling_relay = service_resources_handle
.overwatch_handle
.relay::<DaSamplingService<_, _, _, _, _, _, _>>()
.relay::<DaSamplingService<_, _, _, _>>()
.await
.expect("Relay connection with SamplingService should succeed");
nomos-services/data-availability/dispersal/Cargo.toml
-4
View File
@@ -14,13 +14,9 @@ kzgrs-backend = { workspace = true }
nomos-core = { workspace = true }
nomos-da-network-core = { workspace = true }
nomos-da-network-service = { workspace = true }
nomos-da-sampling = { workspace = true }
nomos-da-verifier = { workspace = true }
nomos-mempool = { workspace = true, features = ["libp2p"] }
nomos-tracing = { workspace = true }
nomos-utils = { workspace = true, features = ["time"] }
overwatch = { workspace = true }
rand = { workspace = true }
serde = { version = "1.0", features = ["derive"] }
serde_with = { workspace = true }
services-utils = { workspace = true }
nomos-services/data-availability/dispersal/src/adapters/mempool/mod.rs
-39
View File
@@ -1,39 +0,0 @@
pub mod kzgrs;
use std::error::Error;
use nomos_mempool::backend::MempoolError;
use overwatch::{
services::{relay::OutboundRelay, ServiceData},
DynError,
};
#[derive(Debug)]
pub enum DaMempoolAdapterError {
Mempool(MempoolError),
Other(DynError),
}
impl<E> From<E> for DaMempoolAdapterError
where
E: Error + Send + Sync + 'static,
{
fn from(e: E) -> Self {
Self::Other(Box::new(e) as DynError)
}
}
#[async_trait::async_trait]
pub trait DaMempoolAdapter {
type MempoolService: ServiceData;
type BlobId;
type Metadata;
fn new(outbound_relay: OutboundRelay<<Self::MempoolService as ServiceData>::Message>) -> Self;
async fn post_blob_id(
&self,
blob_id: Self::BlobId,
metadata: Self::Metadata,
) -> Result<(), DaMempoolAdapterError>;
}
nomos-services/data-availability/dispersal/src/adapters/mod.rs
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod mempool;
pub mod network;
pub mod wallet;
nomos-services/data-availability/dispersal/src/adapters/network/libp2p.rs
+19 -3
View File
@@ -2,7 +2,7 @@ use std::{collections::HashSet, fmt::Debug, marker::PhantomData, pin::Pin, time:
use futures::{stream::BoxStream, Stream, StreamExt as _};
use kzgrs_backend::common::share::{DaShare, DaSharesCommitments};
use nomos_core::da::BlobId;
use nomos_core::{da::BlobId, mantle::SignedMantleTx};
use nomos_da_network_core::{
protocols::{
dispersal::executor::behaviour::DispersalExecutorEvent, sampling::errors::SamplingError,
@@ -127,14 +127,14 @@ where
}
}
async fn disperse(
async fn disperse_share(
&self,
subnetwork_id: Self::SubnetworkId,
da_share: DaShare,
) -> Result<(), DynError> {
self.outbound_relay
.send(DaNetworkMsg::Process(
ExecutorDaNetworkMessage::RequestDispersal {
ExecutorDaNetworkMessage::RequestShareDispersal {
subnetwork_id,
da_share: Box::new(da_share),
},
@@ -143,6 +143,22 @@ where
.map_err(|(e, _)| Box::new(e) as DynError)
}
async fn disperse_tx(
&self,
subnetwork_id: Self::SubnetworkId,
tx: SignedMantleTx,
) -> Result<(), DynError> {
self.outbound_relay
.send(DaNetworkMsg::Process(
ExecutorDaNetworkMessage::RequestTxDispersal {
subnetwork_id,
tx: Box::new(tx),
},
))
.await
.map_err(|(e, _)| Box::new(e) as DynError)
}
async fn dispersal_events_stream(
&self,
) -> Result<
nomos-services/data-availability/dispersal/src/adapters/network/mod.rs
+8 -2
View File
@@ -3,7 +3,7 @@ use std::{pin::Pin, time::Duration};
use futures::Stream;
use kzgrs_backend::common::share::DaShare;
use nomos_core::da::BlobId;
use nomos_core::{da::BlobId, mantle::SignedMantleTx};
use nomos_da_network_core::SubnetworkId;
use overwatch::{
services::{relay::OutboundRelay, ServiceData},
@@ -16,12 +16,18 @@ pub trait DispersalNetworkAdapter {
type SubnetworkId;
fn new(outbound_relay: OutboundRelay<<Self::NetworkService as ServiceData>::Message>) -> Self;
async fn disperse(
async fn disperse_share(
&self,
subnetwork_id: Self::SubnetworkId,
da_share: DaShare,
) -> Result<(), DynError>;
async fn disperse_tx(
&self,
subnetwork_id: Self::SubnetworkId,
tx: SignedMantleTx,
) -> Result<(), DynError>;
async fn dispersal_events_stream(
&self,
) -> Result<
nomos-services/data-availability/dispersal/src/adapters/wallet/mock.rs
+55
View File
@@ -0,0 +1,55 @@
use std::convert::Infallible;
use nomos_core::{
da::BlobId,
mantle::{
ledger::Tx as LedgerTx,
ops::channel::{blob::BlobOp, ChannelId, Ed25519PublicKey, MsgId},
MantleTx, Op, SignedMantleTx, Transaction as _,
},
proofs::zksig::{DummyZkSignature, ZkSignaturePublic},
};
use super::DaWalletAdapter;
pub struct MockWalletAdapter;
impl DaWalletAdapter for MockWalletAdapter {
type Error = Infallible;
fn new() -> Self {
Self
}
fn blob_tx(
&self,
blob: BlobId,
blob_size: usize,
signer: Ed25519PublicKey,
) -> Result<SignedMantleTx, Self::Error> {
let blob_op = BlobOp {
channel: ChannelId::from([0; 32]),
blob,
blob_size: blob_size as u64,
da_storage_gas_price: 0,
parent: MsgId::root(),
signer,
};
let mantle_tx = MantleTx {
ops: vec![Op::ChannelBlob(blob_op)],
ledger_tx: LedgerTx::new(vec![], vec![]),
storage_gas_price: 0,
execution_gas_price: 0,
};
Ok(SignedMantleTx {
ops_profs: Vec::new(),
ledger_tx_proof: DummyZkSignature::prove(ZkSignaturePublic {
msg_hash: mantle_tx.hash().into(),
pks: vec![],
}),
mantle_tx,
})
}
}
nomos-services/data-availability/dispersal/src/adapters/wallet/mod.rs
+21
View File
@@ -0,0 +1,21 @@
pub mod mock;
use nomos_core::{
da::BlobId,
mantle::{ops::channel::Ed25519PublicKey, SignedMantleTx},
};
#[async_trait::async_trait]
pub trait DaWalletAdapter {
type Error;
// TODO: Pass relay when wallet service is defined.
fn new() -> Self;
fn blob_tx(
&self,
blob_id: BlobId,
blob_size: usize,
signer: Ed25519PublicKey,
) -> Result<SignedMantleTx, Self::Error>;
}
nomos-services/data-availability/dispersal/src/backend/kzgrs.rs
+113 -135
View File
@@ -1,27 +1,24 @@
use std::{sync::Arc, time::Duration};
use std::{error::Error, sync::Arc, time::Duration};
use futures::StreamExt as _;
use kzgrs_backend::{
common::build_blob_id,
dispersal, encoder,
encoder,
encoder::{DaEncoderParams, EncodedData},
};
use nomos_core::da::{BlobId, DaDispersal, DaEncoder};
use nomos_mempool::backend::MempoolError;
use nomos_core::{
da::{BlobId, DaDispersal, DaEncoder},
mantle::ops::channel::Ed25519PublicKey,
};
use nomos_tracing::info_with_id;
use nomos_utils::bounded_duration::{MinimalBoundedDuration, NANO};
use overwatch::DynError;
use rand::{seq::IteratorRandom as _, thread_rng};
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
use tokio::time::error::Elapsed;
use tracing::instrument;
use crate::{
adapters::{
mempool::{DaMempoolAdapter, DaMempoolAdapterError},
network::DispersalNetworkAdapter,
},
adapters::{network::DispersalNetworkAdapter, wallet::DaWalletAdapter},
backend::DispersalBackend,
};
@@ -35,20 +32,6 @@ pub struct SampleSubnetworks {
pub cooldown: Duration,
}
#[serde_as]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Timeout {
#[serde_as(as = "MinimalBoundedDuration<1, NANO>")]
pub wait_duration: Duration,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum MempoolPublishStrategy {
Immediately,
Timeout(Timeout),
SampleSubnetworks(SampleSubnetworks),
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct EncoderSettings {
pub num_columns: usize,
@@ -62,43 +45,41 @@ pub struct DispersalKZGRSBackendSettings {
pub encoder_settings: EncoderSettings,
#[serde_as(as = "MinimalBoundedDuration<1, NANO>")]
pub dispersal_timeout: Duration,
pub mempool_strategy: MempoolPublishStrategy,
}
pub struct DispersalKZGRSBackend<NetworkAdapter, MempoolAdapter> {
pub struct DispersalKZGRSBackend<NetworkAdapter, WalletAdapter> {
settings: DispersalKZGRSBackendSettings,
network_adapter: Arc<NetworkAdapter>,
mempool_adapter: MempoolAdapter,
wallet_adapter: Arc<WalletAdapter>,
encoder: Arc<encoder::DaEncoder>,
}
pub struct DispersalFromAdapter<Adapter> {
adapter: Arc<Adapter>,
pub struct DispersalHandler<NetworkAdapter, WalletAdapter> {
network_adapter: Arc<NetworkAdapter>,
wallet_adapter: Arc<WalletAdapter>,
timeout: Duration,
}
#[expect(
dependency_on_unit_never_type_fallback,
reason = "TODO: Remove if solved, this occurs in the timeout method below (out of our handling)"
)]
#[async_trait::async_trait]
impl<Adapter> DaDispersal for DispersalFromAdapter<Adapter>
impl<NetworkAdapter, WalletAdapter> DaDispersal for DispersalHandler<NetworkAdapter, WalletAdapter>
where
Adapter: DispersalNetworkAdapter + Send + Sync,
Adapter::SubnetworkId: From<u16> + Send + Sync,
NetworkAdapter: DispersalNetworkAdapter + Send + Sync,
NetworkAdapter::SubnetworkId: From<u16> + Send + Sync,
WalletAdapter: DaWalletAdapter + Send + Sync,
WalletAdapter::Error: Error + Send + Sync + 'static,
{
type EncodedData = EncodedData;
type Error = DynError;
async fn disperse(&self, encoded_data: Self::EncodedData) -> Result<(), Self::Error> {
let adapter = self.adapter.as_ref();
async fn disperse_shares(&self, encoded_data: Self::EncodedData) -> Result<(), Self::Error> {
let adapter = self.network_adapter.as_ref();
let num_columns = encoded_data.combined_column_proofs.len();
let blob_id = build_blob_id(&encoded_data.row_commitments);
let responses_stream = adapter.dispersal_events_stream().await?;
for (subnetwork_id, share) in encoded_data.into_iter().enumerate() {
adapter
.disperse((subnetwork_id as u16).into(), share)
.disperse_share((subnetwork_id as u16).into(), share)
.await?;
}
@@ -110,7 +91,43 @@ where
}
})
.take(num_columns)
.collect();
.collect::<()>();
// timeout when collecting positive responses
tokio::time::timeout(self.timeout, valid_responses)
.await
.map_err(|e| Box::new(e) as DynError)?;
Ok(())
}
async fn disperse_tx(
&self,
blob_id: BlobId,
num_columns: usize,
original_size: usize,
signer: Ed25519PublicKey,
) -> Result<(), Self::Error> {
let wallet_adapter = self.wallet_adapter.as_ref();
let network_adapter = self.network_adapter.as_ref();
let tx = wallet_adapter
.blob_tx(blob_id, original_size, signer)
.map_err(Box::new)?;
let responses_stream = network_adapter.dispersal_events_stream().await?;
for subnetwork_id in 0..num_columns {
network_adapter
.disperse_tx((subnetwork_id as u16).into(), tx.clone())
.await?;
}
let valid_responses = responses_stream
.filter_map(|event| async move {
match event {
Ok((_blob_id, _)) if _blob_id == blob_id => Some(()),
_ => None,
}
})
.take(num_columns)
.collect::<()>();
// timeout when collecting positive responses
tokio::time::timeout(self.timeout, valid_responses)
.await
@@ -119,26 +136,70 @@ where
}
}
#[async_trait::async_trait]
impl<NetworkAdapter, MempoolAdapter> DispersalBackend
for DispersalKZGRSBackend<NetworkAdapter, MempoolAdapter>
impl<NetworkAdapter, WalletAdapter> DispersalKZGRSBackend<NetworkAdapter, WalletAdapter>
where
NetworkAdapter: DispersalNetworkAdapter + Send + Sync,
NetworkAdapter::SubnetworkId: From<u16> + Send + Sync,
MempoolAdapter: DaMempoolAdapter<BlobId = BlobId, Metadata = dispersal::Metadata> + Send + Sync,
WalletAdapter: DaWalletAdapter + Send + Sync,
WalletAdapter::Error: Error + Send + Sync + 'static,
{
async fn encode(
&self,
data: Vec<u8>,
) -> Result<(BlobId, <encoder::DaEncoder as DaEncoder>::EncodedData), DynError> {
let encoder = Arc::clone(&self.encoder);
// this is a REALLY heavy task, so we should try not to block the thread here
let heavy_task = tokio::task::spawn_blocking(move || encoder.encode(&data));
let encoded_data = heavy_task.await??;
let blob_id = build_blob_id(&encoded_data.row_commitments);
Ok((blob_id, encoded_data))
}
async fn disperse(
&self,
encoded_data: <encoder::DaEncoder as DaEncoder>::EncodedData,
original_size: usize,
) -> Result<(), DynError> {
let blob_id = build_blob_id(&encoded_data.row_commitments);
let num_columns = encoded_data.combined_column_proofs.len();
let handler = DispersalHandler {
network_adapter: Arc::clone(&self.network_adapter),
wallet_adapter: Arc::clone(&self.wallet_adapter),
timeout: self.settings.dispersal_timeout,
};
let () = handler
.disperse_tx(
blob_id,
num_columns,
original_size,
Ed25519PublicKey::from_bytes(&[0u8; 32])?, // TODO: pass key from config
)
.await?;
handler.disperse_shares(encoded_data).await
}
}
#[async_trait::async_trait]
impl<NetworkAdapter, WalletAdapter> DispersalBackend
for DispersalKZGRSBackend<NetworkAdapter, WalletAdapter>
where
NetworkAdapter: DispersalNetworkAdapter + Send + Sync,
NetworkAdapter::SubnetworkId: From<u16> + Send + Sync,
WalletAdapter: DaWalletAdapter + Send + Sync,
WalletAdapter::Error: Error + Send + Sync + 'static,
{
type Settings = DispersalKZGRSBackendSettings;
type Encoder = encoder::DaEncoder;
type Dispersal = DispersalFromAdapter<NetworkAdapter>;
type Dispersal = DispersalHandler<NetworkAdapter, WalletAdapter>;
type NetworkAdapter = NetworkAdapter;
type MempoolAdapter = MempoolAdapter;
type Metadata = dispersal::Metadata;
type WalletAdapter = WalletAdapter;
type BlobId = BlobId;
fn init(
settings: Self::Settings,
network_adapter: Self::NetworkAdapter,
mempool_adapter: Self::MempoolAdapter,
wallet_adapter: Self::WalletAdapter,
) -> Self {
let encoder_settings = &settings.encoder_settings;
let global_params = kzgrs_backend::global::global_parameters_from_file(
@@ -153,100 +214,17 @@ where
Self {
settings,
network_adapter: Arc::new(network_adapter),
mempool_adapter,
wallet_adapter: Arc::new(wallet_adapter),
encoder: Arc::new(encoder),
}
}
async fn encode(
&self,
data: Vec<u8>,
) -> Result<(Self::BlobId, <Self::Encoder as DaEncoder>::EncodedData), DynError> {
let encoder = Arc::clone(&self.encoder);
// this is a REALLY heavy task, so we should try not to block the thread here
let heavy_task = tokio::task::spawn_blocking(move || encoder.encode(&data));
let encoded_data = heavy_task.await??;
let blob_id = build_blob_id(&encoded_data.row_commitments);
Ok((blob_id, encoded_data))
}
async fn disperse(
&self,
encoded_data: <Self::Encoder as DaEncoder>::EncodedData,
) -> Result<(), DynError> {
DispersalFromAdapter {
adapter: Arc::clone(&self.network_adapter),
timeout: self.settings.dispersal_timeout,
}
.disperse(encoded_data)
.await
}
async fn publish_to_mempool(
&self,
blob_id: Self::BlobId,
metadata: Self::Metadata,
) -> Result<(), DynError> {
self.mempool_adapter
.post_blob_id(blob_id, metadata)
.await
.or_else(|err| match err {
DaMempoolAdapterError::Mempool(MempoolError::ExistingItem) => Ok(()),
DaMempoolAdapterError::Mempool(MempoolError::DynamicPoolError(err))
| DaMempoolAdapterError::Other(err) => Err(err),
})
}
#[instrument(skip_all)]
async fn process_dispersal(
&self,
data: Vec<u8>,
metadata: Self::Metadata,
) -> Result<Self::BlobId, DynError> {
async fn process_dispersal(&self, data: Vec<u8>) -> Result<Self::BlobId, DynError> {
let original_size = data.len();
let (blob_id, encoded_data) = self.encode(data).await?;
info_with_id!(blob_id.as_ref(), "ProcessDispersal");
self.disperse(encoded_data).await?;
match self.settings.mempool_strategy {
MempoolPublishStrategy::Immediately => {
self.publish_to_mempool(blob_id, metadata).await?;
}
// MempoolPublishStrategy::Timeout { wait_duration } => {
MempoolPublishStrategy::Timeout(Timeout { wait_duration }) => {
tokio::time::sleep(wait_duration).await;
self.publish_to_mempool(blob_id, metadata).await?;
}
MempoolPublishStrategy::SampleSubnetworks(SampleSubnetworks {
sample_threshold,
timeout,
cooldown,
}) => {
let subnets = {
// ThreadRng is not Send, need to drop before await bound.
let mut rng = thread_rng();
(0..self.settings.encoder_settings.num_columns as u16)
.choose_multiple(&mut rng, sample_threshold)
};
match tokio::time::timeout(
timeout,
self.network_adapter
.get_blob_samples(blob_id, &subnets, cooldown),
)
.await
{
Ok(Ok(())) => {
self.publish_to_mempool(blob_id, metadata).await?;
}
Ok(Err(e)) => return Err(e),
Err(Elapsed { .. }) => {
return Err(Box::new(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("Dispersed blob sampling timed out for blob_id {blob_id:?}"),
)));
}
}
}
}
self.disperse(encoded_data, original_size).await?;
Ok(blob_id)
}
}
nomos-services/data-availability/dispersal/src/backend/mod.rs
+5 -37
View File
@@ -1,11 +1,7 @@
use std::{fmt::Debug, time::Duration};
use nomos_core::da::{blob::metadata, DaDispersal, DaEncoder};
use nomos_tracing::info_with_id;
use nomos_core::da::{DaDispersal, DaEncoder};
use overwatch::DynError;
use tracing::instrument;
use crate::adapters::{mempool::DaMempoolAdapter, network::DispersalNetworkAdapter};
use crate::adapters::{network::DispersalNetworkAdapter, wallet::DaWalletAdapter};
pub mod kzgrs;
@@ -15,42 +11,14 @@ pub trait DispersalBackend {
type Encoder: DaEncoder;
type Dispersal: DaDispersal<EncodedData = <Self::Encoder as DaEncoder>::EncodedData>;
type NetworkAdapter: DispersalNetworkAdapter;
type MempoolAdapter: DaMempoolAdapter;
type Metadata: Debug + metadata::Metadata + Send;
type WalletAdapter: DaWalletAdapter;
type BlobId: AsRef<[u8]> + Send + Copy;
fn init(
config: Self::Settings,
network_adapter: Self::NetworkAdapter,
mempool_adapter: Self::MempoolAdapter,
wallet_adapter: Self::WalletAdapter,
) -> Self;
async fn encode(
&self,
data: Vec<u8>,
) -> Result<(Self::BlobId, <Self::Encoder as DaEncoder>::EncodedData), DynError>;
async fn disperse(
&self,
encoded_data: <Self::Encoder as DaEncoder>::EncodedData,
) -> Result<(), DynError>;
async fn publish_to_mempool(
&self,
blob_id: Self::BlobId,
metadata: Self::Metadata,
) -> Result<(), DynError>;
#[instrument(skip_all)]
async fn process_dispersal(
&self,
data: Vec<u8>,
metadata: Self::Metadata,
) -> Result<Self::BlobId, DynError> {
let (blob_id, encoded_data) = self.encode(data).await?;
info_with_id!(blob_id.as_ref(), "ProcessDispersal");
self.disperse(encoded_data).await?;
// let disperse and replication happen before pushing to mempool
tokio::time::sleep(Duration::from_secs(1)).await;
self.publish_to_mempool(blob_id, metadata).await?;
Ok(blob_id)
}
async fn process_dispersal(&self, data: Vec<u8>) -> Result<Self::BlobId, DynError>;
}
nomos-services/data-availability/dispersal/src/lib.rs
+32 -44
View File
@@ -4,7 +4,7 @@ use std::{
time::Duration,
};
use nomos_core::da::blob::metadata;
use adapters::wallet::{mock::MockWalletAdapter, DaWalletAdapter};
use nomos_da_network_core::{PeerId, SubnetworkId};
use overwatch::{
services::{
@@ -19,19 +19,15 @@ use subnetworks_assignations::MembershipHandler;
use tokio::sync::oneshot;
use tracing::error;
use crate::{
adapters::{mempool::DaMempoolAdapter, network::DispersalNetworkAdapter},
backend::DispersalBackend,
};
use crate::{adapters::network::DispersalNetworkAdapter, backend::DispersalBackend};
pub mod adapters;
pub mod backend;
#[derive(Debug)]
pub enum DaDispersalMsg<Metadata, B: DispersalBackend> {
pub enum DaDispersalMsg<B: DispersalBackend> {
Disperse {
data: Vec<u8>,
metadata: Metadata,
reply_channel: oneshot::Sender<Result<B::BlobId, DynError>>,
},
}
@@ -41,12 +37,20 @@ pub struct DispersalServiceSettings<BackendSettings> {
pub backend: BackendSettings,
}
pub struct DispersalService<
pub type DispersalService<Backend, NetworkAdapter, Membership, RuntimeServiceId> =
GenericDispersalService<
Backend,
NetworkAdapter,
MockWalletAdapter,
Membership,
RuntimeServiceId,
>;
pub struct GenericDispersalService<
Backend,
NetworkAdapter,
MempoolAdapter,
WalletAdapter,
Membership,
Metadata,
RuntimeServiceId,
> where
Membership: MembershipHandler<NetworkId = SubnetworkId, Id = PeerId>
@@ -55,24 +59,22 @@ pub struct DispersalService<
+ Send
+ Sync
+ 'static,
Backend: DispersalBackend<NetworkAdapter = NetworkAdapter, Metadata = Metadata>,
Backend: DispersalBackend<NetworkAdapter = NetworkAdapter>,
Backend::BlobId: Serialize,
Backend::Settings: Clone,
NetworkAdapter: DispersalNetworkAdapter,
MempoolAdapter: DaMempoolAdapter,
Metadata: metadata::Metadata + Debug + 'static,
WalletAdapter: DaWalletAdapter,
{
service_resources_handle: OpaqueServiceResourcesHandle<Self, RuntimeServiceId>,
_backend: PhantomData<Backend>,
}
impl<Backend, NetworkAdapter, MempoolAdapter, Membership, Metadata, RuntimeServiceId> ServiceData
for DispersalService<
impl<Backend, NetworkAdapter, WalletAdapter, Membership, RuntimeServiceId> ServiceData
for GenericDispersalService<
Backend,
NetworkAdapter,
MempoolAdapter,
WalletAdapter,
Membership,
Metadata,
RuntimeServiceId,
>
where
@@ -82,28 +84,26 @@ where
+ Send
+ Sync
+ 'static,
Backend: DispersalBackend<NetworkAdapter = NetworkAdapter, Metadata = Metadata>,
Backend: DispersalBackend<NetworkAdapter = NetworkAdapter>,
Backend::BlobId: Serialize,
Backend::Settings: Clone,
NetworkAdapter: DispersalNetworkAdapter,
MempoolAdapter: DaMempoolAdapter,
Metadata: metadata::Metadata + Debug + 'static,
WalletAdapter: DaWalletAdapter,
{
type Settings = DispersalServiceSettings<Backend::Settings>;
type State = NoState<Self::Settings>;
type StateOperator = NoOperator<Self::State>;
type Message = DaDispersalMsg<Metadata, Backend>;
type Message = DaDispersalMsg<Backend>;
}
#[async_trait::async_trait]
impl<Backend, NetworkAdapter, MempoolAdapter, Membership, Metadata, RuntimeServiceId>
impl<Backend, NetworkAdapter, WalletAdapter, Membership, RuntimeServiceId>
ServiceCore<RuntimeServiceId>
for DispersalService<
for GenericDispersalService<
Backend,
NetworkAdapter,
MempoolAdapter,
WalletAdapter,
Membership,
Metadata,
RuntimeServiceId,
>
where
@@ -113,26 +113,20 @@ where
+ Send
+ Sync
+ 'static,
Backend: DispersalBackend<
NetworkAdapter = NetworkAdapter,
MempoolAdapter = MempoolAdapter,
Metadata = Metadata,
> + Send
Backend: DispersalBackend<NetworkAdapter = NetworkAdapter, WalletAdapter = WalletAdapter>
+ Send
+ Sync,
Backend::Settings: Clone + Send + Sync,
Backend::BlobId: Serialize,
NetworkAdapter: DispersalNetworkAdapter<SubnetworkId = Membership::NetworkId> + Send,
<NetworkAdapter::NetworkService as ServiceData>::Message: 'static,
MempoolAdapter: DaMempoolAdapter,
<MempoolAdapter::MempoolService as ServiceData>::Message: 'static,
Metadata: metadata::Metadata + Debug + Send + 'static,
WalletAdapter: DaWalletAdapter + Send,
RuntimeServiceId: Debug
+ Sync
+ Display
+ Send
+ AsServiceId<Self>
+ AsServiceId<NetworkAdapter::NetworkService>
+ AsServiceId<MempoolAdapter::MempoolService>
+ 'static,
{
fn init(
@@ -162,12 +156,8 @@ where
.relay::<NetworkAdapter::NetworkService>()
.await?;
let network_adapter = NetworkAdapter::new(network_relay);
let mempool_relay = service_resources_handle
.overwatch_handle
.relay::<MempoolAdapter::MempoolService>()
.await?;
let mempool_adapter = MempoolAdapter::new(mempool_relay);
let backend = Backend::init(backend_settings, network_adapter, mempool_adapter);
let wallet_adapter = WalletAdapter::new();
let backend = Backend::init(backend_settings, network_adapter, wallet_adapter);
let mut inbound_relay = service_resources_handle.inbound_relay;
service_resources_handle.status_updater.notify_ready();
@@ -179,8 +169,7 @@ where
wait_until_services_are_ready!(
&service_resources_handle.overwatch_handle,
Some(Duration::from_secs(60)),
NetworkAdapter::NetworkService,
MempoolAdapter::MempoolService
NetworkAdapter::NetworkService
)
.await?;
@@ -188,10 +177,9 @@ where
match dispersal_msg {
DaDispersalMsg::Disperse {
data,
metadata,
reply_channel,
} => {
let response = backend.process_dispersal(data, metadata).await;
let response = backend.process_dispersal(data).await;
if let Err(Err(e)) = reply_channel.send(response) {
error!("Error forwarding dispersal response: {e}");
}
nomos-services/data-availability/indexer/Cargo.toml
-1
View File
@@ -15,7 +15,6 @@ futures = "0.3"
kzgrs-backend = { workspace = true }
nomos-core = { workspace = true }
nomos-da-sampling = { workspace = true }
nomos-da-verifier = { workspace = true }
nomos-mempool = { workspace = true }
nomos-storage = { workspace = true }
nomos-time = { workspace = true }
nomos-services/data-availability/indexer/src/lib.rs
+2 -46
View File
@@ -53,9 +53,6 @@ pub struct DataIndexerService<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
> where
@@ -88,11 +85,6 @@ pub struct DataIndexerService<
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
{
@@ -139,9 +131,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
> ServiceData
@@ -161,9 +150,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>
@@ -197,11 +183,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
{
@@ -227,9 +208,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>
@@ -249,9 +227,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>
@@ -287,11 +262,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
{
@@ -350,9 +320,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
> ServiceCore<RuntimeServiceId>
@@ -372,9 +339,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>
@@ -435,11 +399,6 @@ where
SamplingBackend::BlobId: Debug + 'static,
SamplingNetworkAdapter: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId>,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId>,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId>,
DaVerifierNetwork::Settings: Clone,
TimeBackend: nomos_time::backends::TimeBackend,
TimeBackend::Settings: Clone + Send + Sync,
RuntimeServiceId: Debug
@@ -461,9 +420,6 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
TimeBackend,
RuntimeServiceId,
>,
@@ -486,7 +442,7 @@ where
let consensus_relay = service_resources_handle
.overwatch_handle
.relay::<CryptarchiaConsensus<_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _>>()
.relay::<CryptarchiaConsensus<_, _, _, _, _, _, _, _, _, _, _, _, _, _>>()
.await
.expect("Relay connection with ConsensusService should succeed");
let storage_relay = service_resources_handle
@@ -509,7 +465,7 @@ where
&service_resources_handle.overwatch_handle,
Some(Duration::from_secs(60)),
StorageService<_, _>,
CryptarchiaConsensus<_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _>
CryptarchiaConsensus<_, _, _, _, _, _, _, _, _, _, _, _, _, _>
)
.await?;
nomos-services/data-availability/network/Cargo.toml
+2
View File
@@ -19,6 +19,7 @@ libp2p-identity = { version = "0.2" }
log = "0.4.22"
multiaddr = "0.18"
nomos-core = { workspace = true }
nomos-da-messages = { workspace = true }
nomos-da-network-core = { workspace = true }
nomos-libp2p = { workspace = true }
nomos-membership = { workspace = true }
@@ -28,6 +29,7 @@ nomos-utils = { workspace = true, features = ["rng"] }
overwatch = { workspace = true }
rand = { workspace = true }
serde = { version = "1.0", features = ["derive"] }
services-utils = { workspace = true }
subnetworks-assignations = { workspace = true }
thiserror = "1"
tokio = { version = "1", features = ["macros", "sync"] }
nomos-services/data-availability/network/src/backends/libp2p/common.rs
+151 -18
View File
@@ -8,28 +8,35 @@ use kzgrs_backend::common::{
share::{DaLightShare, DaShare, DaSharesCommitments},
ShareIndex,
};
use nomos_core::{block::BlockNumber, da::BlobId, header::HeaderId};
use nomos_core::{block::BlockNumber, da::BlobId, header::HeaderId, mantle::SignedMantleTx};
use nomos_da_messages::common::Share;
use nomos_da_network_core::{
maintenance::{balancer::ConnectionBalancerCommand, monitor::ConnectionMonitorCommand},
protocols::sampling::{
self, errors::SamplingError, BehaviourSampleReq, BehaviourSampleRes, SubnetsConfig,
protocols::{
dispersal::validator::behaviour::DispersalEvent,
sampling::{
self, errors::SamplingError, BehaviourSampleReq, BehaviourSampleRes, SubnetsConfig,
},
},
swarm::{
validator::{SampleArgs, ValidatorEventsStream},
DAConnectionMonitorSettings, DAConnectionPolicySettings, ReplicationConfig,
DAConnectionMonitorSettings, DAConnectionPolicySettings, DispersalValidationError,
DispersalValidationResult, DispersalValidatorEvent, ReplicationConfig,
},
};
use nomos_libp2p::{ed25519, secret_key_serde, Multiaddr};
use serde::{Deserialize, Serialize};
use tokio::sync::{
broadcast, mpsc,
mpsc::{error::SendError, UnboundedSender},
broadcast,
mpsc::{self, error::SendError, UnboundedSender},
oneshot,
};
use tracing::error;
pub(crate) const BROADCAST_CHANNEL_SIZE: usize = 128;
pub type BroadcastValidationResultSender = Option<mpsc::Sender<DispersalValidationResult>>;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DaNetworkBackendSettings {
// Identification Secp256k1 private key in Hex format (`0x123...abc`). Default random.
@@ -42,7 +49,6 @@ pub struct DaNetworkBackendSettings {
pub redial_cooldown: Duration,
pub replication_settings: ReplicationConfig,
pub subnets_settings: SubnetsConfig,
pub refresh_interval: Duration,
}
/// Sampling events coming from da network
@@ -97,12 +103,55 @@ impl SamplingEvent {
}
}
#[derive(Debug, Clone)]
pub enum VerificationEvent {
Tx {
// Number of subnetwork assignations that the node is assigned to at the moment of
// receiving a dispersal TX. This is added to the TX message received via the network
// because of dynamic assignations that can change depending
// on the DA session, and syncinc these moments preciselly between services even if some
// TX events might reach destination after the assignations has already changed.
assignations: u16,
tx: Box<SignedMantleTx>,
response_sender: BroadcastValidationResultSender,
},
Share {
share: Box<DaShare>,
response_sender: BroadcastValidationResultSender,
},
}
impl From<(DaShare, BroadcastValidationResultSender)> for VerificationEvent {
fn from((share, response_sender): (DaShare, BroadcastValidationResultSender)) -> Self {
Self::Share {
share: Box::new(share),
response_sender,
}
}
}
impl From<(u16, Box<SignedMantleTx>, BroadcastValidationResultSender)> for VerificationEvent {
fn from(
(assignations, tx, response_sender): (
u16,
Box<SignedMantleTx>,
BroadcastValidationResultSender,
),
) -> Self {
Self::Tx {
assignations,
tx,
response_sender,
}
}
}
/// Task that handles forwarding of events to the subscriptions channels/stream
pub(crate) async fn handle_validator_events_stream(
events_streams: ValidatorEventsStream,
sampling_broadcast_sender: broadcast::Sender<SamplingEvent>,
commitments_broadcast_sender: broadcast::Sender<CommitmentsEvent>,
validation_broadcast_sender: broadcast::Sender<DaShare>,
validation_broadcast_sender: broadcast::Sender<VerificationEvent>,
) {
let ValidatorEventsStream {
mut sampling_events_receiver,
@@ -114,18 +163,102 @@ pub(crate) async fn handle_validator_events_stream(
// safe set: https://docs.rs/tokio/latest/tokio/macro.select.html#cancellation-safety
tokio::select! {
Some(sampling_event) = StreamExt::next(&mut sampling_events_receiver) => {
handle_event(&sampling_broadcast_sender, &commitments_broadcast_sender, sampling_event).await;
handle_sampling_event(&sampling_broadcast_sender, &commitments_broadcast_sender, sampling_event).await;
}
Some(da_share) = StreamExt::next(&mut validation_events_receiver) => {
if let Err(error) = validation_broadcast_sender.send(da_share) {
error!("Error in internal broadcast of validation for blob: {:?}", error.0);
}
Some(dispersal_event) = StreamExt::next(&mut validation_events_receiver) => {
handle_dispersal_event(&validation_broadcast_sender, dispersal_event).await;
}
}
}
}
async fn handle_event(
#[expect(
clippy::cognitive_complexity,
reason = "complex comunication between libp2p behaviour and service"
)]
async fn handle_dispersal_event(
validation_broadcast_sender: &broadcast::Sender<VerificationEvent>,
dispersal_event: DispersalValidatorEvent,
) {
match (dispersal_event.event, dispersal_event.sender) {
(DispersalEvent::IncomingShare(share), Some(sender)) => {
handle_incoming_share_with_response(validation_broadcast_sender, sender, share).await;
}
(DispersalEvent::IncomingShare(share), None) => {
if let Err(error) = validation_broadcast_sender.send((share.data, None).into()) {
error!(
"Error in internal broadcast of validation for blob: {:?}",
error.0
);
}
}
(DispersalEvent::IncomingTx((assignations, tx)), Some(sender)) => {
handle_incoming_tx_with_response(validation_broadcast_sender, sender, assignations, tx)
.await;
}
(DispersalEvent::IncomingTx((assignations, tx)), None) => {
if let Err(error) = validation_broadcast_sender.send((assignations, tx, None).into()) {
error!(
"Error in internal broadcast of validation for blob: {:?}",
error.0
);
}
}
(DispersalEvent::DispersalError { error }, _) => {
error!("Error from dispersal behaviour: {error:?}");
}
}
}
async fn handle_incoming_share_with_response(
validation_broadcast_sender: &broadcast::Sender<VerificationEvent>,
behaviour_sender: Sender<DispersalValidationResult>,
share: Box<Share>,
) {
let (service_sender, mut service_receiver) =
mpsc::channel::<DispersalValidationResult>(BROADCAST_CHANNEL_SIZE);
if let Err(error) = validation_broadcast_sender.send((share.data, Some(service_sender)).into())
{
let _ = behaviour_sender.send(Err(DispersalValidationError));
error!(
"Error in internal broadcast of validation for blob: {:?}",
error.0
);
return;
}
let validation_response = service_receiver
.recv()
.await
.unwrap_or(Err(DispersalValidationError));
let _ = behaviour_sender.send(validation_response);
}
async fn handle_incoming_tx_with_response(
validation_broadcast_sender: &broadcast::Sender<VerificationEvent>,
behaviour_sender: Sender<DispersalValidationResult>,
assignations: u16,
tx: Box<SignedMantleTx>,
) {
let (service_sender, mut service_receiver) =
mpsc::channel::<DispersalValidationResult>(BROADCAST_CHANNEL_SIZE);
if let Err(error) =
validation_broadcast_sender.send((assignations, tx, Some(service_sender)).into())
{
let _ = behaviour_sender.send(Err(DispersalValidationError));
error!(
"Error in internal broadcast of validation for blob: {:?}",
error.0
);
return;
}
let validation_response = service_receiver
.recv()
.await
.unwrap_or(Err(DispersalValidationError));
let _ = behaviour_sender.send(validation_response);
}
async fn handle_sampling_event(
sampling_broadcast_sender: &broadcast::Sender<SamplingEvent>,
commitments_broadcast_sender: &broadcast::Sender<CommitmentsEvent>,
sampling_event: sampling::SamplingEvent,
@@ -160,7 +293,7 @@ async fn handle_event(
request_receiver,
response_sender,
} => {
handle_request(
handle_sampling_request(
sampling_broadcast_sender,
commitments_broadcast_sender,
request_receiver,
@@ -169,7 +302,7 @@ async fn handle_event(
.await;
}
sampling::SamplingEvent::SamplingError { error } => {
handle_error(
handle_sampling_error(
sampling_broadcast_sender,
commitments_broadcast_sender,
error,
@@ -182,7 +315,7 @@ async fn handle_event(
}
}
fn handle_error(
fn handle_sampling_error(
sampling_broadcast_sender: &broadcast::Sender<SamplingEvent>,
commitments_broadcast_sender: &broadcast::Sender<CommitmentsEvent>,
error: SamplingError,
@@ -198,7 +331,7 @@ fn handle_error(
}
}
async fn handle_request(
async fn handle_sampling_request(
sampling_broadcast_sender: &broadcast::Sender<SamplingEvent>,
commitments_broadcast_sender: &broadcast::Sender<CommitmentsEvent>,
request_receiver: Receiver<BehaviourSampleReq>,
nomos-services/data-availability/network/src/backends/libp2p/executor.rs
+23 -21
View File
@@ -7,7 +7,7 @@ use futures::{
use kzgrs_backend::common::share::DaShare;
use libp2p::PeerId;
use log::error;
use nomos_core::{block::BlockNumber, da::BlobId, header::HeaderId};
use nomos_core::{block::BlockNumber, da::BlobId, header::HeaderId, mantle::SignedMantleTx};
use nomos_da_network_core::{
maintenance::{balancer::ConnectionBalancerCommand, monitor::ConnectionMonitorCommand},
protocols::dispersal::executor::behaviour::DispersalExecutorEvent,
@@ -23,14 +23,11 @@ use nomos_tracing::info_with_id;
use overwatch::{overwatch::handle::OverwatchHandle, services::state::NoState};
use serde::{Deserialize, Serialize};
use subnetworks_assignations::MembershipHandler;
use tokio::{
sync::{broadcast, mpsc::UnboundedSender, oneshot},
time,
};
use tokio_stream::wrappers::{BroadcastStream, IntervalStream, UnboundedReceiverStream};
use tokio::sync::{broadcast, mpsc::UnboundedSender, oneshot};
use tokio_stream::wrappers::{BroadcastStream, UnboundedReceiverStream};
use tracing::instrument;
use super::common::CommitmentsEvent;
use super::common::{CommitmentsEvent, VerificationEvent};
use crate::{
backends::{
libp2p::common::{
@@ -54,10 +51,14 @@ pub enum ExecutorDaNetworkMessage<BalancerStats, MonitorStats> {
RequestCommitments {
blob_id: BlobId,
},
RequestDispersal {
RequestShareDispersal {
subnetwork_id: SubnetworkId,
da_share: Box<DaShare>,
},
RequestTxDispersal {
subnetwork_id: SubnetworkId,
tx: Box<SignedMantleTx>,
},
MonitorRequest(ConnectionMonitorCommand<MonitorStats>),
BalancerStats(oneshot::Sender<BalancerStats>),
}
@@ -78,7 +79,7 @@ pub enum DaNetworkEventKind {
pub enum DaNetworkEvent {
Sampling(SamplingEvent),
Commitments(CommitmentsEvent),
Verifying(Box<DaShare>),
Verifying(VerificationEvent),
Dispersal(DispersalExecutorEvent),
}
@@ -104,9 +105,10 @@ where
commitments_request_channel: UnboundedSender<BlobId>,
sampling_broadcast_receiver: broadcast::Receiver<SamplingEvent>,
commitments_broadcast_receiver: broadcast::Receiver<CommitmentsEvent>,
verifying_broadcast_receiver: broadcast::Receiver<DaShare>,
verifying_broadcast_receiver: broadcast::Receiver<VerificationEvent>,
dispersal_broadcast_receiver: broadcast::Receiver<DispersalExecutorEvent>,
dispersal_shares_sender: UnboundedSender<(Membership::NetworkId, DaShare)>,
dispersal_tx_sender: UnboundedSender<(Membership::NetworkId, SignedMantleTx)>,
balancer_command_sender: UnboundedSender<ConnectionBalancerCommand<BalancerStats>>,
monitor_command_sender: UnboundedSender<ConnectionMonitorCommand<MonitorStats>>,
_membership: PhantomData<Membership>,
@@ -138,15 +140,8 @@ where
overwatch_handle: OverwatchHandle<RuntimeServiceId>,
membership: Self::Membership,
addressbook: Self::Addressbook,
subnet_refresh_signal: impl Stream<Item = ()> + Send + 'static,
) -> Self {
// TODO: If there is no requirement to subscribe to block number events in chain
// service, and an approximate duration is enough for sampling to hold
// temporal connections - remove this message.
let subnet_refresh_signal = Box::pin(
IntervalStream::new(time::interval(config.validator_settings.refresh_interval))
.map(|_| ()),
);
let keypair = libp2p::identity::Keypair::from(ed25519::Keypair::from(
config.validator_settings.node_key.clone(),
));
@@ -177,6 +172,7 @@ where
let historic_sample_request_channel = executor_swarm.historic_sample_request_channel();
let commitments_request_channel = executor_swarm.commitments_request_channel();
let dispersal_shares_sender = executor_swarm.dispersal_shares_channel();
let dispersal_tx_sender = executor_swarm.dispersal_tx_channel();
let balancer_command_sender = executor_swarm.balancer_command_channel();
let monitor_command_sender = executor_swarm.monitor_command_channel();
@@ -228,6 +224,7 @@ where
verifying_broadcast_receiver,
dispersal_broadcast_receiver,
dispersal_shares_sender,
dispersal_tx_sender,
balancer_command_sender,
monitor_command_sender,
_membership: PhantomData,
@@ -257,11 +254,11 @@ where
info_with_id!(&blob_id, "RequestSample");
handle_commitments_request(&self.commitments_request_channel, blob_id).await;
}
ExecutorDaNetworkMessage::RequestDispersal {
ExecutorDaNetworkMessage::RequestShareDispersal {
subnetwork_id,
da_share,
} => {
info_with_id!(&da_share.blob_id(), "RequestDispersal");
info_with_id!(&da_share.blob_id(), "RequestShareDispersal");
if let Err(e) = self
.dispersal_shares_sender
.send((subnetwork_id, *da_share))
@@ -269,6 +266,11 @@ where
error!("Could not send internal blob to underlying dispersal behaviour: {e}");
}
}
ExecutorDaNetworkMessage::RequestTxDispersal { subnetwork_id, tx } => {
if let Err(e) = self.dispersal_tx_sender.send((subnetwork_id, *tx)) {
error!("Could not send internal tx to underlying dispersal behaviour: {e}");
}
}
ExecutorDaNetworkMessage::MonitorRequest(command) => {
match command.peer_id() {
Some(peer_id) => {
@@ -305,7 +307,7 @@ where
DaNetworkEventKind::Verifying => Box::pin(
BroadcastStream::new(self.verifying_broadcast_receiver.resubscribe())
.filter_map(|event| async { event.ok() })
.map(|share| Self::NetworkEvent::Verifying(Box::new(share))),
.map(Self::NetworkEvent::Verifying),
),
DaNetworkEventKind::Dispersal => Box::pin(
BroadcastStream::new(self.dispersal_broadcast_receiver.resubscribe())
nomos-services/data-availability/network/src/backends/libp2p/validator.rs
+8 -18
View File
@@ -4,7 +4,6 @@ use futures::{
future::{AbortHandle, Abortable},
Stream, StreamExt as _,
};
use kzgrs_backend::common::share::DaShare;
use libp2p::PeerId;
use nomos_core::{block::BlockNumber, da::BlobId, header::HeaderId};
use nomos_da_network_core::{
@@ -20,20 +19,16 @@ use nomos_tracing::info_with_id;
use overwatch::{overwatch::handle::OverwatchHandle, services::state::NoState};
use serde::Serialize;
use subnetworks_assignations::MembershipHandler;
use tokio::{
sync::{broadcast, mpsc::UnboundedSender, oneshot},
time,
};
use tokio_stream::wrappers::{BroadcastStream, IntervalStream};
use tokio::sync::{broadcast, mpsc::UnboundedSender, oneshot};
use tokio_stream::wrappers::BroadcastStream;
use tracing::instrument;
use super::common::CommitmentsEvent;
use crate::{
backends::{
libp2p::common::{
handle_balancer_command, handle_historic_sample_request, handle_monitor_command,
handle_sample_request, handle_validator_events_stream, DaNetworkBackendSettings,
SamplingEvent, BROADCAST_CHANNEL_SIZE,
handle_sample_request, handle_validator_events_stream, CommitmentsEvent,
DaNetworkBackendSettings, SamplingEvent, VerificationEvent, BROADCAST_CHANNEL_SIZE,
},
NetworkBackend,
},
@@ -70,7 +65,7 @@ pub enum DaNetworkEventKind {
pub enum DaNetworkEvent {
Sampling(SamplingEvent),
Commitments(CommitmentsEvent),
Verifying(Box<DaShare>),
Verifying(VerificationEvent),
}
/// DA network backend for validators
@@ -86,7 +81,7 @@ pub struct DaNetworkValidatorBackend<Membership> {
monitor_command_sender: UnboundedSender<ConnectionMonitorCommand<MonitorStats>>,
sampling_broadcast_receiver: broadcast::Receiver<SamplingEvent>,
commitments_broadcast_receiver: broadcast::Receiver<CommitmentsEvent>,
verifying_broadcast_receiver: broadcast::Receiver<DaShare>,
verifying_broadcast_receiver: broadcast::Receiver<VerificationEvent>,
_membership: PhantomData<Membership>,
}
@@ -116,13 +111,8 @@ where
overwatch_handle: OverwatchHandle<RuntimeServiceId>,
membership: Self::Membership,
addressbook: Self::Addressbook,
subnet_refresh_signal: impl Stream<Item = ()> + Send + 'static,
) -> Self {
// TODO: If there is no requirement to subscribe to block number events in chain
// service, and an approximate duration is enough for sampling to hold
// temporal connections - remove this message.
let subnet_refresh_signal =
Box::pin(IntervalStream::new(time::interval(config.refresh_interval)).map(|_| ()));
let keypair =
libp2p::identity::Keypair::from(ed25519::Keypair::from(config.node_key.clone()));
let (mut validator_swarm, validator_events_stream) = ValidatorSwarm::new(
@@ -243,7 +233,7 @@ where
DaNetworkEventKind::Verifying => Box::pin(
BroadcastStream::new(self.verifying_broadcast_receiver.resubscribe())
.filter_map(|event| async { event.ok() })
.map(|share| Self::NetworkEvent::Verifying(Box::new(share))),
.map(Self::NetworkEvent::Verifying),
),
}
}
nomos-services/data-availability/network/src/backends/mock/executor.rs
+1
View File
@@ -83,6 +83,7 @@ impl<RuntimeServiceId> NetworkBackend<RuntimeServiceId> for MockExecutorBackend
_: OverwatchHandle<RuntimeServiceId>,
_membership: Self::Membership,
_addressbook: Self::Addressbook,
_subnet_refresh_signal: impl Stream<Item = ()> + Send + 'static,
) -> Self {
let (commands_tx, _) = mpsc::channel(BUFFER_SIZE);
let (events_tx, _) = broadcast::channel(BUFFER_SIZE);
nomos-services/data-availability/network/src/backends/mod.rs
+1
View File
@@ -27,6 +27,7 @@ pub trait NetworkBackend<RuntimeServiceId> {
overwatch_handle: OverwatchHandle<RuntimeServiceId>,
membership: Self::Membership,
addressbook: Self::Addressbook,
subnet_refresh_signal: impl Stream<Item = ()> + Send + 'static,
) -> Self;
fn shutdown(&mut self);
async fn process(&self, msg: Self::Message);
nomos-services/data-availability/network/src/lib.rs
+35 -4
View File
@@ -9,11 +9,12 @@ use std::{
fmt::{self, Debug, Display},
marker::PhantomData,
pin::Pin,
time::Duration,
};
use async_trait::async_trait;
use backends::NetworkBackend;
use futures::Stream;
use futures::{stream::select, Stream};
use kzgrs_backend::common::share::{DaShare, DaSharesCommitments};
use libp2p::{Multiaddr, PeerId};
use nomos_core::{block::BlockNumber, da::BlobId, header::HeaderId};
@@ -26,10 +27,17 @@ use overwatch::{
OpaqueServiceResourcesHandle,
};
use serde::{Deserialize, Serialize};
use services_utils::wait_until_services_are_ready;
use storage::{MembershipStorage, MembershipStorageAdapter};
use subnetworks_assignations::{MembershipCreator, MembershipHandler, SubnetworkAssignations};
use tokio::sync::oneshot;
use tokio_stream::StreamExt as _;
use tokio::sync::{
mpsc::{self, Sender},
oneshot,
};
use tokio_stream::{
wrappers::{IntervalStream, ReceiverStream},
StreamExt as _,
};
use crate::{
addressbook::{AddressBook, AddressBookSnapshot},
@@ -116,6 +124,7 @@ pub struct NetworkConfig<
pub backend: Backend::Settings,
pub membership: Membership,
pub api_adapter_settings: ApiAdapterSettings,
pub subnet_refresh_interval: Duration,
}
impl<
@@ -150,6 +159,7 @@ pub struct NetworkService<
addressbook: DaAddressbook,
api_adapter: ApiAdapter,
phantom: PhantomData<MembershipServiceAdapter>,
subnet_refresh_sender: Sender<()>,
}
pub struct NetworkState<
@@ -256,7 +266,8 @@ where
+ Sync
+ Debug
+ AsServiceId<MembershipServiceAdapter::MembershipService>
+ AsServiceId<StorageAdapter::StorageService>,
+ AsServiceId<StorageAdapter::StorageService>
+ 'static,
{
fn init(
service_resources_handle: OpaqueServiceResourcesHandle<Self, RuntimeServiceId>,
@@ -276,18 +287,28 @@ where
addressbook.clone(),
);
// Sampling subnetwork peers need to be updatedd periodically.
// They also need to be updated when the assignations change.
let (subnet_refresh_sender, refresh_rx) = mpsc::channel(1);
let interval = tokio::time::interval(settings.subnet_refresh_interval);
let refresh_ticker = IntervalStream::new(interval).map(|_| ());
let refresh_signal = ReceiverStream::new(refresh_rx);
let subnet_refresh_signal = select(refresh_ticker, refresh_signal);
Ok(Self {
backend: <Backend as NetworkBackend<RuntimeServiceId>>::new(
settings.backend,
service_resources_handle.overwatch_handle.clone(),
membership.clone(),
addressbook.clone(),
subnet_refresh_signal,
),
service_resources_handle,
membership,
addressbook,
api_adapter,
phantom: PhantomData,
subnet_refresh_sender,
})
}
@@ -304,6 +325,7 @@ where
ref membership,
ref api_adapter,
ref addressbook,
ref subnet_refresh_sender,
..
} = self;
@@ -321,6 +343,13 @@ where
let membership_service_adapter = MembershipServiceAdapter::new(membership_service_relay);
wait_until_services_are_ready!(
&self.service_resources_handle.overwatch_handle,
Some(Duration::from_secs(60)),
<MembershipServiceAdapter as MembershipAdapter>::MembershipService
)
.await?;
let mut stream = membership_service_adapter.subscribe().await.map_err(|e| {
tracing::error!("Failed to subscribe to membership service: {e}");
e
@@ -343,6 +372,7 @@ where
block_number, providers
);
Self::handle_membership_update(block_number, providers, &membership_storage).await;
let _ = subnet_refresh_sender.send(()).await;
}
}
}
@@ -548,6 +578,7 @@ where
backend: self.backend.clone(),
membership: self.membership.clone(),
api_adapter_settings: self.api_adapter_settings.clone(),
subnet_refresh_interval: self.subnet_refresh_interval,
}
}
}
nomos-services/data-availability/sampling/Cargo.toml
-1
View File
@@ -17,7 +17,6 @@ libp2p-identity = { version = "0.2" }
nomos-core = { workspace = true }
nomos-da-network-core = { workspace = true }
nomos-da-network-service = { workspace = true }
nomos-da-verifier = { workspace = true }
nomos-storage = { workspace = true }
nomos-tracing = { workspace = true }
overwatch = { workspace = true }
nomos-services/data-availability/sampling/src/lib.rs
+51 -136
View File
@@ -1,32 +1,25 @@
pub mod backend;
pub mod network;
pub mod storage;
pub mod verifier;
use std::{
collections::BTreeSet,
fmt::{Debug, Display},
marker::PhantomData,
sync::Arc,
time::Duration,
};
use backend::{DaSamplingServiceBackend, SamplingState};
use kzgrs_backend::common::share::{DaLightShare, DaShare, DaSharesCommitments};
use kzgrs_backend::common::share::{DaShare, DaSharesCommitments};
use network::NetworkAdapter;
use nomos_core::da::{blob::Share, BlobId, DaVerifier};
use nomos_core::da::BlobId;
use nomos_da_network_core::protocols::sampling::errors::SamplingError;
use nomos_da_network_service::{
backends::libp2p::common::SamplingEvent, membership::MembershipAdapter,
storage::MembershipStorageAdapter, NetworkService,
};
use nomos_da_verifier::{
backend::VerifierBackend as VerifierBackendTrait, DaVerifierMsg, DaVerifierService,
};
use nomos_da_network_service::{backends::libp2p::common::SamplingEvent, NetworkService};
use nomos_storage::StorageService;
use nomos_tracing::{error_with_id, info_with_id};
use overwatch::{
services::{
relay::OutboundRelay,
state::{NoOperator, NoState},
AsServiceId, ServiceCore, ServiceData,
},
@@ -39,17 +32,16 @@ use subnetworks_assignations::MembershipHandler;
use tokio::sync::oneshot;
use tokio_stream::StreamExt as _;
use tracing::{error, instrument};
use verifier::{kzgrs::KzgrsDaVerifier, VerifierBackend};
type VerifierRelay<DaVerifierBackend> = OutboundRelay<
DaVerifierMsg<
<<DaVerifierBackend as DaVerifier>::DaShare as Share>::SharesCommitments,
<<DaVerifierBackend as DaVerifier>::DaShare as Share>::LightShare,
<DaVerifierBackend as DaVerifier>::DaShare,
(),
>,
>;
type VerifierMessage = DaVerifierMsg<DaSharesCommitments, DaLightShare, DaShare, ()>;
pub type DaSamplingService<SamplingBackend, SamplingNetwork, SamplingStorage, RuntimeServiceId> =
GenericDaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
KzgrsDaVerifier,
RuntimeServiceId,
>;
#[derive(Debug)]
pub enum DaSamplingServiceMsg<BlobId> {
@@ -65,56 +57,45 @@ pub enum DaSamplingServiceMsg<BlobId> {
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DaSamplingServiceSettings<BackendSettings> {
pub struct DaSamplingServiceSettings<BackendSettings, ShareVerifierSettings> {
pub sampling_settings: BackendSettings,
pub share_verifier_settings: ShareVerifierSettings,
}
pub struct DaSamplingService<
pub struct GenericDaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
ShareVerifier,
RuntimeServiceId,
> where
SamplingBackend: DaSamplingServiceBackend,
SamplingNetwork: NetworkAdapter<RuntimeServiceId>,
SamplingStorage: DaStorageAdapter<RuntimeServiceId>,
ShareVerifier: VerifierBackend,
{
service_resources_handle: OpaqueServiceResourcesHandle<Self, RuntimeServiceId>,
_phantom: PhantomData<(
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
ShareVerifier,
)>,
}
impl<
impl<SamplingBackend, SamplingNetwork, SamplingStorage, ShareVerifier, RuntimeServiceId>
GenericDaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
RuntimeServiceId,
>
DaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
ShareVerifier,
RuntimeServiceId,
>
where
SamplingBackend: DaSamplingServiceBackend,
SamplingNetwork: NetworkAdapter<RuntimeServiceId>,
SamplingStorage: DaStorageAdapter<RuntimeServiceId>,
ShareVerifier: VerifierBackend,
{
#[must_use]
pub const fn new(
@@ -127,22 +108,12 @@ where
}
}
impl<
impl<SamplingBackend, SamplingNetwork, SamplingStorage, ShareVerifier, RuntimeServiceId>
GenericDaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
RuntimeServiceId,
>
DaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
ShareVerifier,
RuntimeServiceId,
>
where
@@ -154,7 +125,7 @@ where
SamplingBackend::Settings: Clone,
SamplingNetwork: NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: DaStorageAdapter<RuntimeServiceId, Share = DaShare> + Send + Sync,
VerifierBackend: VerifierBackendTrait<DaShare = DaShare>,
ShareVerifier: VerifierBackend<DaShare = DaShare> + Send + Sync,
{
#[instrument(skip_all)]
async fn handle_service_message(
@@ -202,7 +173,7 @@ where
event: SamplingEvent,
sampler: &mut SamplingBackend,
storage_adapter: &SamplingStorage,
verifier_relay: &VerifierRelay<VerifierBackend>,
verifier: &ShareVerifier,
) {
match event {
SamplingEvent::SamplingSuccess {
@@ -215,10 +186,7 @@ where
sampler.handle_sampling_error(blob_id).await;
return;
};
if Self::verify_blob(verifier_relay, commitments, light_share.clone())
.await
.is_ok()
{
if verifier.verify(&commitments, &light_share).is_ok() {
sampler
.handle_sampling_success(blob_id, light_share.share_idx)
.await;
@@ -278,74 +246,36 @@ where
.ok()
.flatten()
}
async fn verify_blob(
verifier_relay: &OutboundRelay<VerifierMessage>,
commitments: Arc<DaSharesCommitments>,
light_share: Box<DaLightShare>,
) -> Result<(), DynError> {
let (reply_sender, reply_channel) = oneshot::channel();
verifier_relay
.send(DaVerifierMsg::VerifyShare {
commitments,
light_share,
reply_channel: reply_sender,
})
.await
.expect("Failed to send verify blob message to verifier relay");
reply_channel
.await
.expect("Failed to receive reply blob message from verifier relay")
}
}
impl<
impl<SamplingBackend, SamplingNetwork, SamplingStorage, ShareVerifier, RuntimeServiceId> ServiceData
for GenericDaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
RuntimeServiceId,
> ServiceData
for DaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
ShareVerifier,
RuntimeServiceId,
>
where
SamplingBackend: DaSamplingServiceBackend,
SamplingNetwork: NetworkAdapter<RuntimeServiceId>,
SamplingStorage: DaStorageAdapter<RuntimeServiceId>,
ShareVerifier: VerifierBackend,
{
type Settings = DaSamplingServiceSettings<SamplingBackend::Settings>;
type Settings = DaSamplingServiceSettings<SamplingBackend::Settings, ShareVerifier::Settings>;
type State = NoState<Self::Settings>;
type StateOperator = NoOperator<Self::State>;
type Message = DaSamplingServiceMsg<SamplingBackend::BlobId>;
}
#[async_trait::async_trait]
impl<
impl<SamplingBackend, SamplingNetwork, SamplingStorage, ShareVerifier, RuntimeServiceId>
ServiceCore<RuntimeServiceId>
for GenericDaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
RuntimeServiceId,
> ServiceCore<RuntimeServiceId>
for DaSamplingService<
SamplingBackend,
SamplingNetwork,
SamplingStorage,
VerifierBackend,
VerifierNetwork,
VerifierStorage,
ShareVerifier,
RuntimeServiceId,
>
where
@@ -359,33 +289,20 @@ where
SamplingNetwork::Settings: Send + Sync,
SamplingNetwork::Membership: MembershipHandler + Clone + 'static,
SamplingStorage: DaStorageAdapter<RuntimeServiceId, Share = DaShare> + Send + Sync,
VerifierBackend:
nomos_da_verifier::backend::VerifierBackend<DaShare = SamplingBackend::Share> + Send,
VerifierBackend::Settings: Clone,
VerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send,
VerifierNetwork::Settings: Clone,
VerifierNetwork::Storage: MembershipStorageAdapter<
<SamplingNetwork::Membership as MembershipHandler>::Id,
<SamplingNetwork::Membership as MembershipHandler>::NetworkId,
> + Send
+ Sync
+ 'static,
VerifierNetwork::MembershipAdapter: MembershipAdapter,
VerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send,
ShareVerifier: VerifierBackend<DaShare = DaShare> + Send + Sync,
ShareVerifier::Settings: Clone + Send + Sync,
RuntimeServiceId: AsServiceId<Self>
+ AsServiceId<
NetworkService<
SamplingNetwork::Backend,
SamplingNetwork::Membership,
VerifierNetwork::MembershipAdapter,
VerifierNetwork::Storage,
VerifierNetwork::ApiAdapter,
SamplingNetwork::MembershipAdapter,
SamplingNetwork::Storage,
SamplingNetwork::ApiAdapter,
RuntimeServiceId,
>,
> + AsServiceId<StorageService<SamplingStorage::Backend, RuntimeServiceId>>
+ AsServiceId<
DaVerifierService<VerifierBackend, VerifierNetwork, VerifierStorage, RuntimeServiceId>,
> + Debug
+ Debug
+ Display
+ Sync
+ Send
@@ -403,7 +320,10 @@ where
mut service_resources_handle,
..
} = self;
let DaSamplingServiceSettings { sampling_settings } = service_resources_handle
let DaSamplingServiceSettings {
sampling_settings,
share_verifier_settings,
} = service_resources_handle
.settings_handle
.notifier()
.get_updated_settings();
@@ -421,12 +341,8 @@ where
.await?;
let storage_adapter = SamplingStorage::new(storage_relay).await;
let verifier_relay = service_resources_handle
.overwatch_handle
.relay::<DaVerifierService<_, _, _, _>>()
.await?;
let mut sampler = SamplingBackend::new(sampling_settings);
let share_verifier = ShareVerifier::new(share_verifier_settings);
let mut next_prune_tick = sampler.prune_interval();
service_resources_handle.status_updater.notify_ready();
@@ -439,8 +355,7 @@ where
&service_resources_handle.overwatch_handle,
Some(Duration::from_secs(60)),
NetworkService<_, _, _, _,_, _>,
StorageService<_, _>,
DaVerifierService<_, _, _, _>
StorageService<_, _>
)
.await?;
@@ -450,7 +365,7 @@ where
Self::handle_service_message(service_message, &mut network_adapter, &storage_adapter, &mut sampler).await;
}
Some(sampling_message) = sampling_message_stream.next() => {
Self::handle_sampling_message(sampling_message, &mut sampler, &storage_adapter, &verifier_relay).await;
Self::handle_sampling_message(sampling_message, &mut sampler, &storage_adapter, &share_verifier).await;
}
// cleanup not on time samples
_ = next_prune_tick.tick() => {
nomos-services/data-availability/sampling/src/storage/adapters/rocksdb/converter.rs
+17 -1
View File
@@ -1,6 +1,9 @@
use bytes::Bytes;
use kzgrs_backend::common::share::{DaLightShare, DaShare, DaSharesCommitments};
use nomos_core::da::{blob::Share, BlobId};
use nomos_core::{
da::{blob::Share, BlobId},
mantle::SignedMantleTx,
};
use nomos_storage::{
api::da::{DaConverter, StorageDaApi},
backends::{rocksdb::RocksBackend, StorageSerde},
@@ -14,6 +17,7 @@ where
<SerdeOP as StorageSerde>::Error: Send + Sync + 'static,
{
type Share = DaShare;
type Tx = SignedMantleTx;
type Error = SerdeOP::Error;
fn blob_id_to_storage(blob_id: BlobId) -> Result<BlobId, Self::Error> {
@@ -55,4 +59,16 @@ where
) -> Result<DaSharesCommitments, Self::Error> {
SerdeOP::deserialize(backend_commitments)
}
fn tx_to_storage(
service_tx: SignedMantleTx,
) -> Result<<RocksBackend<SerdeOP> as StorageDaApi>::Tx, Self::Error> {
Ok(SerdeOP::serialize(&service_tx))
}
fn tx_from_storage(
backend_tx: <RocksBackend<SerdeOP> as StorageDaApi>::Tx,
) -> Result<SignedMantleTx, Self::Error> {
SerdeOP::deserialize(backend_tx)
}
}
nomos-services/data-availability/sampling/src/verifier/kzgrs.rs
+69
View File
@@ -0,0 +1,69 @@
use core::fmt;
use kzgrs_backend::{
common::share::DaShare, global::global_parameters_from_file,
verifier::DaVerifier as NomosKzgrsVerifier,
};
use nomos_core::da::{blob::Share, DaVerifier};
use serde::{Deserialize, Serialize};
use super::VerifierBackend;
#[derive(Debug)]
pub enum KzgrsDaVerifierError {
VerificationError,
}
impl fmt::Display for KzgrsDaVerifierError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
Self::VerificationError => write!(f, "Verification failed"),
}
}
}
impl std::error::Error for KzgrsDaVerifierError {}
pub struct KzgrsDaVerifier {
verifier: NomosKzgrsVerifier,
domain_size: usize,
}
impl VerifierBackend for KzgrsDaVerifier {
type Settings = KzgrsDaVerifierSettings;
fn new(settings: Self::Settings) -> Self {
let global_params = global_parameters_from_file(&settings.global_params_path)
.expect("Global parameters has to be loaded from file");
let verifier = NomosKzgrsVerifier::new(global_params);
Self {
verifier,
domain_size: settings.domain_size,
}
}
}
impl DaVerifier for KzgrsDaVerifier {
type DaShare = DaShare;
type Error = KzgrsDaVerifierError;
fn verify(
&self,
commitments: &<Self::DaShare as Share>::SharesCommitments,
light_share: &<Self::DaShare as Share>::LightShare,
) -> Result<(), Self::Error> {
// TODO: Prepare the domain depending the size, if fixed, so fixed domain, if
// not it needs to come with some metadata.
self.verifier
.verify(light_share, commitments, self.domain_size)
.then_some(())
.ok_or(KzgrsDaVerifierError::VerificationError)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KzgrsDaVerifierSettings {
pub global_params_path: String,
pub domain_size: usize,
}
nomos-services/data-availability/sampling/src/verifier/mod.rs
+8
View File
@@ -0,0 +1,8 @@
use nomos_core::da::DaVerifier;
pub mod kzgrs;
pub trait VerifierBackend: DaVerifier {
type Settings;
fn new(settings: Self::Settings) -> Self;
}
nomos-services/data-availability/verifier/Cargo.toml
+5
View File
@@ -15,12 +15,17 @@ libp2p = { workspace = true, features = ["ed25519"] }
nomos-core = { workspace = true }
nomos-da-network-core = { workspace = true }
nomos-da-network-service = { workspace = true }
nomos-da-sampling = { workspace = true }
nomos-mempool = { workspace = true, features = ["libp2p", "mock"] }
nomos-storage = { workspace = true }
nomos-tracing = { workspace = true }
nomos-utils = { workspace = true, features = ["time"] }
overwatch = { workspace = true }
serde = { version = "1.0", features = ["derive"] }
serde_with = { workspace = true }
services-utils = { workspace = true }
subnetworks-assignations = { workspace = true }
thiserror = "1.0"
tokio = { version = "1", features = ["macros", "sync"] }
tokio-stream = "0.1.15"
tracing = { workspace = true, features = ["attributes"] }
nomos-services/data-availability/verifier/src/backend/mod.rs
+13
View File
@@ -1,4 +1,6 @@
pub mod kzgrs;
pub mod trigger;
pub mod tx;
pub use nomos_core::da::DaVerifier;
@@ -6,3 +8,14 @@ pub trait VerifierBackend: DaVerifier {
type Settings;
fn new(settings: Self::Settings) -> Self;
}
pub trait TxVerifierBackend {
type Settings;
type Tx;
type BlobId;
type Error;
fn new(settings: Self::Settings) -> Self;
fn verify(&self, tx: &Self::Tx) -> Result<(), Self::Error>;
fn blob_id(&self, tx: &Self::Tx) -> Result<Self::BlobId, Self::Error>;
}
nomos-services/data-availability/verifier/src/backend/trigger.rs
+294
View File
@@ -0,0 +1,294 @@
use std::{
collections::HashMap,
hash::Hash,
sync::{
atomic::{AtomicBool, AtomicU16, Ordering},
Arc, RwLock,
},
time::{Duration, Instant},
};
use nomos_utils::{
bounded_duration::{MinimalBoundedDuration, NANO, SECOND},
math::NonNegativeF64,
};
use serde::{Deserialize, Serialize};
use serde_with::serde_as;
#[serde_as]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct MempoolPublishTriggerConfig {
/// A percentage of shares required for the transaction to be published
/// after the a `share_duration`.
pub publish_threshold: NonNegativeF64,
/// Duration after which the transaction should be published if
/// `publish_threshold` is reached, or marked as expired if not reached.
#[serde_as(as = "MinimalBoundedDuration<1, NANO>")]
pub share_duration: Duration,
/// A period after which expired states are removed from memory.
#[serde_as(as = "MinimalBoundedDuration<1, NANO>")]
pub prune_duration: Duration,
/// An interval for pruning expired states.
#[serde_as(as = "MinimalBoundedDuration<1, SECOND>")]
pub prune_interval: Duration,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ShareState {
Complete,
Incomplete,
Expired,
}
#[derive(Debug)]
struct ShareEntry {
count: AtomicU16,
created_at: Instant,
assignations: u16,
expired: AtomicBool,
}
pub struct MempoolPublishTrigger<Id> {
config: MempoolPublishTriggerConfig,
received: Arc<RwLock<HashMap<Id, Arc<ShareEntry>>>>,
}
impl<Id: Clone + Hash + Eq> MempoolPublishTrigger<Id> {
#[must_use]
pub fn new(config: MempoolPublishTriggerConfig) -> Self {
Self {
config,
received: Arc::new(RwLock::new(HashMap::new())),
}
}
pub fn update(&self, blob_id: Id, assignations: u16) -> ShareState {
let maybe_entry = self.received.read().unwrap().get(&blob_id).cloned();
let entry = maybe_entry.unwrap_or_else(|| {
let mut map = self.received.write().unwrap();
Arc::clone(map.entry(blob_id).or_insert_with(|| {
Arc::new(ShareEntry {
count: AtomicU16::new(0),
created_at: Instant::now(),
assignations,
expired: AtomicBool::new(false),
})
}))
});
if entry.expired.load(Ordering::Acquire) {
return ShareState::Expired;
}
let new_count = entry.count.fetch_add(1, Ordering::AcqRel) + 1;
if new_count >= entry.assignations {
ShareState::Complete
} else {
ShareState::Incomplete
}
}
#[must_use]
pub fn prune(&self, now: Instant) -> Vec<Id> {
let mut to_publish = Vec::new();
let mut map = self.received.write().unwrap();
map.retain(|blob_id, state| {
let elapsed = now.duration_since(state.created_at);
if elapsed >= self.config.prune_duration {
return false;
}
if !state.expired.load(Ordering::Acquire) && elapsed >= self.config.share_duration {
state.expired.store(true, Ordering::Release);
let count = state.count.load(Ordering::Acquire);
let threshold = (f64::from(state.assignations)
* self.config.publish_threshold.get())
.ceil() as u16;
if count >= threshold {
to_publish.push(blob_id.clone());
}
}
true
});
to_publish
}
}
#[cfg(test)]
mod tests {
use std::thread;
use super::*;
const BLOB_ID: [u8; 32] = [1; 32];
impl<Id: Hash + Eq> MempoolPublishTrigger<Id> {
fn len(&self) -> usize {
self.received.read().unwrap().len()
}
fn get_count(&self, blob_id: &Id) -> Option<u16> {
self.received
.read()
.unwrap()
.get(blob_id)
.map(|e| e.count.load(Ordering::Relaxed))
}
}
fn test_config() -> MempoolPublishTriggerConfig {
MempoolPublishTriggerConfig {
publish_threshold: NonNegativeF64::try_from(0.5).unwrap(),
share_duration: Duration::from_secs(5),
prune_duration: Duration::from_secs(10),
prune_interval: Duration::from_secs(5),
}
}
#[test]
fn test_update_reaches_complete() {
let trigger = MempoolPublishTrigger::new(test_config());
let assignations = 3;
assert_eq!(
trigger.update(BLOB_ID, assignations),
ShareState::Incomplete
);
assert_eq!(trigger.get_count(&BLOB_ID), Some(1));
assert_eq!(
trigger.update(BLOB_ID, assignations),
ShareState::Incomplete
);
assert_eq!(trigger.get_count(&BLOB_ID), Some(2));
assert_eq!(trigger.update(BLOB_ID, assignations), ShareState::Complete);
assert_eq!(trigger.get_count(&BLOB_ID), Some(3));
// Additional updates should still report Complete
assert_eq!(trigger.update(BLOB_ID, assignations), ShareState::Complete);
assert_eq!(trigger.get_count(&BLOB_ID), Some(4));
}
#[test]
fn test_prune_and_publish() {
let trigger = MempoolPublishTrigger::new(test_config());
let now = Instant::now();
// 6 out of 10 shares received (60%).
for _ in 0..6 {
trigger.update(BLOB_ID, 10);
}
let later = now.checked_add(Duration::from_secs(6)).unwrap();
let to_publish = trigger.prune(later);
assert_eq!(to_publish, vec![BLOB_ID]);
assert_eq!(trigger.len(), 1);
}
#[test]
fn test_prune_and_not_publish() {
let trigger = MempoolPublishTrigger::new(test_config());
let now = Instant::now();
// 4 out of 10 shares received (40%), which is < 50% threshold.
for _ in 0..4 {
trigger.update(BLOB_ID, 10);
}
let later = now.checked_add(test_config().share_duration).unwrap();
let to_publish = trigger.prune(later);
assert!(to_publish.is_empty());
assert_eq!(trigger.len(), 1);
}
#[test]
fn test_prune_removes_old_entry() {
let trigger = MempoolPublishTrigger::new(test_config());
let now = Instant::now();
trigger.update(BLOB_ID, 10);
assert_eq!(trigger.len(), 1);
let later = now.checked_add(Duration::from_secs(11)).unwrap();
let to_publish = trigger.prune(later);
assert!(to_publish.is_empty());
assert_eq!(trigger.len(), 0);
}
#[test]
fn test_update_on_expired_entry() {
let now = Instant::now();
let trigger = MempoolPublishTrigger::new(test_config());
trigger.update(BLOB_ID, 10);
let later = now.checked_add(Duration::from_secs(6)).unwrap();
let _ = trigger.prune(later);
assert_eq!(trigger.update(BLOB_ID, 10), ShareState::Expired);
}
#[test]
fn test_prune_publish_is_once() {
let now = Instant::now();
let trigger = MempoolPublishTrigger::new(test_config());
for _ in 0..5 {
trigger.update(BLOB_ID, 10);
}
let later = now.checked_add(Duration::from_secs(6)).unwrap();
assert_eq!(trigger.prune(later), vec![BLOB_ID]);
assert!(trigger.prune(later).is_empty());
}
#[test]
fn test_concurrent_updates() {
let assignations = 100;
let trigger = Arc::new(MempoolPublishTrigger::new(test_config()));
let mut handles = vec![];
for _ in 0..assignations {
let trigger_clone = Arc::clone(&trigger);
handles.push(thread::spawn(move || {
trigger_clone.update(BLOB_ID, assignations as u16)
}));
}
let mut complete_count = 0;
let mut incomplete_count = 0;
for handle in handles {
match handle.join().unwrap() {
ShareState::Complete => complete_count += 1,
ShareState::Incomplete => incomplete_count += 1,
ShareState::Expired => panic!("Should not be expired"),
}
}
assert_eq!(
complete_count, 1,
"Exactly one update should return Complete"
);
assert_eq!(
incomplete_count,
assignations - 1,
"The rest should be Incomplete"
);
assert_eq!(
trigger.get_count(&BLOB_ID),
Some(assignations as u16),
"Final count should match number of updates"
);
}
}
nomos-services/data-availability/verifier/src/backend/tx/mock.rs
+41
View File
@@ -0,0 +1,41 @@
use nomos_core::{
da::BlobId,
mantle::{
ops::{channel::blob::BlobOp, Op},
SignedMantleTx,
},
};
use thiserror::Error;
use crate::backend::TxVerifierBackend;
#[derive(Error, Debug)]
pub enum MockTxVerifierError {
#[error("Transaction has no blob id")]
NoBlobId,
}
pub struct MockTxVerifier;
impl TxVerifierBackend for MockTxVerifier {
type Settings = ();
type Tx = SignedMantleTx;
type BlobId = BlobId;
type Error = MockTxVerifierError;
fn new(_settings: Self::Settings) -> Self {
Self
}
fn verify(&self, _tx: &Self::Tx) -> Result<(), Self::Error> {
Ok(())
}
fn blob_id(&self, tx: &Self::Tx) -> Result<Self::BlobId, Self::Error> {
if let Some(Op::ChannelBlob(BlobOp { blob, .. })) = tx.mantle_tx.ops.first() {
Ok(*blob)
} else {
Err(MockTxVerifierError::NoBlobId)
}
}
}
nomos-services/data-availability/verifier/src/backend/tx/mod.rs
+1
View File
@@ -0,0 +1 @@
pub mod mock;
nomos-services/data-availability/verifier/src/lib.rs
+272 -63
View File
@@ -1,22 +1,31 @@
pub mod backend;
pub mod mempool;
pub mod network;
pub mod storage;
use std::{
error::Error,
fmt::{Debug, Display, Formatter},
hash::Hash,
sync::Arc,
time::Duration,
time::{Duration, Instant},
};
use backend::VerifierBackend;
use backend::{
trigger::{MempoolPublishTrigger, MempoolPublishTriggerConfig},
tx::mock::MockTxVerifier,
TxVerifierBackend, VerifierBackend,
};
use mempool::{DaMempoolAdapter, MempoolAdapterError};
use network::NetworkAdapter;
use nomos_core::da::blob::Share;
use nomos_da_network_core::swarm::DispersalValidationError;
use nomos_da_network_service::{
membership::MembershipAdapter, storage::MembershipStorageAdapter, NetworkService,
};
use nomos_mempool::backend::MempoolError;
use nomos_storage::StorageService;
use nomos_tracing::info_with_id;
use nomos_tracing::{error_with_id, info_with_id};
use overwatch::{
services::{
state::{NoOperator, NoState},
@@ -32,6 +41,18 @@ use tokio::sync::oneshot::Sender;
use tokio_stream::StreamExt as _;
use tracing::{error, instrument};
use crate::network::ValidationRequest;
pub type DaVerifierService<ShareVerifier, Network, Storage, MempoolAdapter, RuntimeServiceId> =
GenericDaVerifierService<
ShareVerifier,
MockTxVerifier,
Network,
Storage,
MempoolAdapter,
RuntimeServiceId,
>;
pub enum DaVerifierMsg<Commitments, LightShare, Share, Answer> {
AddShare {
share: Share,
@@ -57,90 +78,203 @@ impl<C: 'static, L: 'static, B: 'static, A: 'static> Debug for DaVerifierMsg<C,
}
}
pub struct DaVerifierService<Backend, Network, Storage, RuntimeServiceId>
where
Backend: VerifierBackend,
Backend::Settings: Clone,
Backend::DaShare: 'static,
pub struct GenericDaVerifierService<
ShareVerifier,
TxVerifier,
Network,
Storage,
MempoolAdapter,
RuntimeServiceId,
> where
ShareVerifier: VerifierBackend,
ShareVerifier::Settings: Clone,
ShareVerifier::DaShare: 'static,
TxVerifier: TxVerifierBackend,
TxVerifier::Settings: Clone,
Network: NetworkAdapter<RuntimeServiceId>,
Network::Settings: Clone,
MempoolAdapter: DaMempoolAdapter,
Storage: DaStorageAdapter<RuntimeServiceId>,
{
service_resources_handle: OpaqueServiceResourcesHandle<Self, RuntimeServiceId>,
verifier: Backend,
share_verifier: ShareVerifier,
tx_verifier: TxVerifier,
}
impl<Backend, Network, Storage, RuntimeServiceId>
DaVerifierService<Backend, Network, Storage, RuntimeServiceId>
impl<ShareVerifier, TxVerifier, Network, Storage, MempoolAdapter, RuntimeServiceId>
GenericDaVerifierService<
ShareVerifier,
TxVerifier,
Network,
Storage,
MempoolAdapter,
RuntimeServiceId,
>
where
Backend: VerifierBackend + Send + Sync + 'static,
Backend::DaShare: Debug + Send,
Backend::Error: Error + Send + Sync,
Backend::Settings: Clone,
<Backend::DaShare as Share>::BlobId: AsRef<[u8]>,
Network: NetworkAdapter<RuntimeServiceId, Share = Backend::DaShare> + Send + 'static,
ShareVerifier: VerifierBackend + Send + Sync + 'static,
ShareVerifier::DaShare: Debug + Send,
ShareVerifier::Error: Error + Send + Sync,
ShareVerifier::Settings: Clone,
<ShareVerifier::DaShare as Share>::BlobId: Clone + AsRef<[u8]> + Hash + Eq + Send + Sync,
<ShareVerifier::DaShare as Share>::LightShare: Send,
<ShareVerifier::DaShare as Share>::SharesCommitments: Send,
TxVerifier: TxVerifierBackend<BlobId = <ShareVerifier::DaShare as Share>::BlobId> + Send + Sync,
TxVerifier::Settings: Clone,
TxVerifier::Tx: Send,
TxVerifier::Error: Error + Send + Sync + 'static,
Network: NetworkAdapter<RuntimeServiceId, Share = ShareVerifier::DaShare, Tx = TxVerifier::Tx>
+ Send
+ 'static,
Network::Settings: Clone,
Storage: DaStorageAdapter<RuntimeServiceId, Share = Backend::DaShare> + Send + Sync + 'static,
MempoolAdapter: DaMempoolAdapter<
BlobId = <ShareVerifier::DaShare as Share>::BlobId,
Tx = <TxVerifier as TxVerifierBackend>::Tx,
> + Send
+ Sync
+ 'static,
Storage: DaStorageAdapter<RuntimeServiceId, Share = ShareVerifier::DaShare, Tx = TxVerifier::Tx>
+ Send
+ Sync
+ 'static,
{
#[instrument(skip_all)]
async fn handle_new_share(
verifier: &Backend,
verifier: &ShareVerifier,
storage_adapter: &Storage,
share: Backend::DaShare,
mempool_trigger: &MempoolPublishTrigger<<ShareVerifier::DaShare as Share>::BlobId>,
mempool_adapter: &MempoolAdapter,
share: ShareVerifier::DaShare,
) -> Result<(), DynError> {
if storage_adapter
.get_share(share.blob_id(), share.share_idx())
.await?
.is_some()
{
info_with_id!(share.blob_id().as_ref(), "VerifierShareExists");
if let Some((assignations, tx)) = storage_adapter.get_tx(share.blob_id()).await? {
if storage_adapter
.get_share(share.blob_id(), share.share_idx())
.await?
.is_some()
{
info_with_id!(share.blob_id().as_ref(), "VerifierShareExists");
} else {
info_with_id!(share.blob_id().as_ref(), "VerifierAddShare");
let (blob_id, share_idx) = (share.blob_id(), share.share_idx());
let (light_share, commitments) = share.into_share_and_commitments();
// TODO: remove TX if verification fails.
verifier.verify(&commitments, &light_share)?;
storage_adapter
.add_share(blob_id.clone(), share_idx, commitments, light_share)
.await?;
if matches!(
mempool_trigger.update(blob_id.clone(), assignations),
backend::trigger::ShareState::Complete
) {
mempool_adapter.post_tx(blob_id, tx).await?;
}
}
} else {
info_with_id!(share.blob_id().as_ref(), "VerifierAddShare");
let (blob_id, share_idx) = (share.blob_id(), share.share_idx());
let (light_share, commitments) = share.into_share_and_commitments();
verifier.verify(&commitments, &light_share)?;
storage_adapter
.add_share(blob_id, share_idx, commitments, light_share)
.await?;
error_with_id!(share.blob_id().as_ref(), "VerifierTxDoesNotExist");
return Err("Transaction doesn't exist".into());
}
Ok(())
}
async fn handle_new_tx(
verifier: &TxVerifier,
storage_adapter: &Storage,
assignations: u16,
tx: TxVerifier::Tx,
) -> Result<(), DynError> {
let blob_id = verifier.blob_id(&tx)?;
if storage_adapter.get_tx(blob_id.clone()).await?.is_some() {
info_with_id!(blob_id.as_ref(), "VerifierTxExists");
} else {
info_with_id!(blob_id.as_ref(), "VerifierAddTx");
verifier.verify(&tx)?;
storage_adapter.add_tx(blob_id, assignations, tx).await?;
}
Ok(())
}
async fn prune_pending_txs(
storage_adapter: &Storage,
mempool_trigger: &MempoolPublishTrigger<<ShareVerifier::DaShare as Share>::BlobId>,
mempool_adapter: &MempoolAdapter,
) -> Result<(), DynError> {
let now = Instant::now();
let blob_ids = mempool_trigger.prune(now);
for blob_id in blob_ids {
if let Some((_, tx)) = storage_adapter.get_tx(blob_id.clone()).await? {
match mempool_adapter.post_tx(blob_id, tx).await {
Ok(()) | Err(MempoolAdapterError::Mempool(MempoolError::ExistingItem)) => {}
Err(err) => return Err(Box::new(err)),
};
}
}
Ok(())
}
}
impl<Backend, Network, DaStorage, RuntimeServiceId> ServiceData
for DaVerifierService<Backend, Network, DaStorage, RuntimeServiceId>
impl<ShareVerifier, TxVerifier, Network, DaStorage, MempoolAdapter, RuntimeServiceId> ServiceData
for GenericDaVerifierService<
ShareVerifier,
TxVerifier,
Network,
DaStorage,
MempoolAdapter,
RuntimeServiceId,
>
where
Backend: VerifierBackend,
Backend::Settings: Clone,
ShareVerifier: VerifierBackend,
ShareVerifier::Settings: Clone,
TxVerifier: TxVerifierBackend,
TxVerifier::Settings: Clone,
Network: NetworkAdapter<RuntimeServiceId>,
Network::Settings: Clone,
DaStorage: DaStorageAdapter<RuntimeServiceId>,
DaStorage::Settings: Clone,
MempoolAdapter: DaMempoolAdapter,
{
type Settings =
DaVerifierServiceSettings<Backend::Settings, Network::Settings, DaStorage::Settings>;
type Settings = DaVerifierServiceSettings<
ShareVerifier::Settings,
TxVerifier::Settings,
Network::Settings,
DaStorage::Settings,
>;
type State = NoState<Self::Settings>;
type StateOperator = NoOperator<Self::State>;
type Message = DaVerifierMsg<
<Backend::DaShare as Share>::SharesCommitments,
<Backend::DaShare as Share>::LightShare,
Backend::DaShare,
<ShareVerifier::DaShare as Share>::SharesCommitments,
<ShareVerifier::DaShare as Share>::LightShare,
ShareVerifier::DaShare,
(),
>;
}
#[async_trait::async_trait]
impl<Backend, Network, DaStorage, RuntimeServiceId> ServiceCore<RuntimeServiceId>
for DaVerifierService<Backend, Network, DaStorage, RuntimeServiceId>
impl<ShareVerifier, TxVerifier, Network, DaStorage, MempoolAdapter, RuntimeServiceId>
ServiceCore<RuntimeServiceId>
for GenericDaVerifierService<
ShareVerifier,
TxVerifier,
Network,
DaStorage,
MempoolAdapter,
RuntimeServiceId,
>
where
Backend: VerifierBackend + Send + Sync + 'static,
Backend::Settings: Clone + Send + Sync + 'static,
Backend::DaShare: Debug + Send + Sync + 'static,
Backend::Error: Error + Send + Sync + 'static,
<Backend::DaShare as Share>::BlobId: AsRef<[u8]> + Debug + Send + Sync + 'static,
<Backend::DaShare as Share>::LightShare: Debug + Send + Sync + 'static,
<Backend::DaShare as Share>::SharesCommitments: Debug + Send + Sync + 'static,
Network: NetworkAdapter<RuntimeServiceId, Share = Backend::DaShare> + Send + Sync + 'static,
ShareVerifier: VerifierBackend + Send + Sync + 'static,
ShareVerifier::Settings: Clone + Send + Sync + 'static,
ShareVerifier::DaShare: Debug + Send + Sync + 'static,
ShareVerifier::Error: Error + Send + Sync + 'static,
<ShareVerifier::DaShare as Share>::BlobId:
Clone + AsRef<[u8]> + Debug + Hash + Eq + Send + Sync + 'static,
<ShareVerifier::DaShare as Share>::LightShare: Debug + Send + Sync + 'static,
<ShareVerifier::DaShare as Share>::SharesCommitments: Debug + Send + Sync + 'static,
TxVerifier: TxVerifierBackend<BlobId = <ShareVerifier::DaShare as Share>::BlobId> + Send + Sync,
TxVerifier::Tx: Send,
TxVerifier::Settings: Clone + Send + Sync + 'static,
TxVerifier::Error: Error + Send + Sync + 'static,
Network: NetworkAdapter<RuntimeServiceId, Share = ShareVerifier::DaShare, Tx = TxVerifier::Tx>
+ Send
+ Sync
+ 'static,
Network::Membership: MembershipHandler + Clone,
Network::Settings: Clone + Send + Sync + 'static,
Network::Storage: MembershipStorageAdapter<
@@ -150,8 +284,17 @@ where
+ Sync
+ 'static,
Network::MembershipAdapter: MembershipAdapter,
DaStorage: DaStorageAdapter<RuntimeServiceId, Share = Backend::DaShare> + Send + Sync + 'static,
DaStorage: DaStorageAdapter<RuntimeServiceId, Share = ShareVerifier::DaShare, Tx = TxVerifier::Tx>
+ Send
+ Sync
+ 'static,
DaStorage::Settings: Clone + Send + Sync + 'static,
MempoolAdapter: DaMempoolAdapter<
BlobId = <ShareVerifier::DaShare as Share>::BlobId,
Tx = <TxVerifier as TxVerifierBackend>::Tx,
> + Send
+ Sync
+ 'static,
RuntimeServiceId: Debug
+ Display
+ Sync
@@ -168,6 +311,7 @@ where
RuntimeServiceId,
>,
>
+ AsServiceId<MempoolAdapter::MempoolService>
+ AsServiceId<StorageService<DaStorage::Backend, RuntimeServiceId>>,
{
fn init(
@@ -175,17 +319,24 @@ where
_initial_state: Self::State,
) -> Result<Self, DynError> {
let DaVerifierServiceSettings {
verifier_settings, ..
share_verifier_settings,
tx_verifier_settings,
..
} = service_resources_handle
.settings_handle
.notifier()
.get_updated_settings();
Ok(Self {
service_resources_handle,
verifier: Backend::new(verifier_settings),
share_verifier: ShareVerifier::new(share_verifier_settings),
tx_verifier: TxVerifier::new(tx_verifier_settings),
})
}
#[expect(
clippy::too_many_lines,
reason = "Run loop contains all handling for readablity"
)]
async fn run(self) -> Result<(), DynError> {
// This service will likely have to be modified later on.
// Most probably the verifier itself need to be constructed/update for every
@@ -194,11 +345,13 @@ where
// in the above-mentioned list.
let Self {
mut service_resources_handle,
verifier,
share_verifier,
tx_verifier,
} = self;
let DaVerifierServiceSettings {
network_adapter_settings,
mempool_trigger_settings,
..
} = service_resources_handle
.settings_handle
@@ -211,6 +364,13 @@ where
.await?;
let network_adapter = Network::new(network_adapter_settings, network_relay).await;
let mut share_stream = network_adapter.share_stream().await;
let mut tx_stream = network_adapter.tx_stream().await;
let mempool_relay = service_resources_handle
.overwatch_handle
.relay::<MempoolAdapter::MempoolService>()
.await?;
let mempool_adapter = MempoolAdapter::new(mempool_relay);
let storage_relay = service_resources_handle
.overwatch_handle
@@ -224,6 +384,9 @@ where
<RuntimeServiceId as AsServiceId<Self>>::SERVICE_ID
);
let mut prune_interval = tokio::time::interval(mempool_trigger_settings.prune_interval);
let mempool_trigger = MempoolPublishTrigger::new(mempool_trigger_settings);
wait_until_services_are_ready!(
&service_resources_handle.overwatch_handle,
Some(Duration::from_secs(60)),
@@ -234,17 +397,48 @@ where
loop {
tokio::select! {
Some(share) = share_stream.next() => {
Some(ValidationRequest{item: share, sender}) = share_stream.next() => {
let blob_id = share.blob_id();
if let Err(err) = Self::handle_new_share(&verifier,&storage_adapter, share).await {
if let Err(err) = Self::handle_new_share(
&share_verifier,
&storage_adapter,
&mempool_trigger,
&mempool_adapter,
share
).await {
if let Some(sender) = sender {
let _ = sender.send(Err(DispersalValidationError)).await;
}
error!("Error handling blob {blob_id:?} due to {err:?}");
continue;
}
if let Some(sender) = sender {
let _ = sender.send(Ok(())).await;
}
}
Some(ValidationRequest{ item: (assignations, tx), sender }) = tx_stream.next() => {
if let Err(err) = Self::handle_new_tx(&tx_verifier, &storage_adapter, assignations, tx).await {
if let Some(sender) = sender {
let _ = sender.send(Err(DispersalValidationError)).await;
}
error!("Error handling tx due to {err:?}");
continue;
}
if let Some(sender) = sender {
let _ = sender.send(Ok(())).await;
}
}
Some(msg) = service_resources_handle.inbound_relay.recv() => {
match msg {
DaVerifierMsg::AddShare { share, reply_channel } => {
let blob_id = share.blob_id();
match Self::handle_new_share(&verifier, &storage_adapter, share).await {
match Self::handle_new_share(
&share_verifier,
&storage_adapter,
&mempool_trigger,
&mempool_adapter,
share
).await {
Ok(attestation) => {
if let Err(err) = reply_channel.send(Some(attestation)) {
error!("Error replying attestation {err:?}");
@@ -259,7 +453,7 @@ where
};
},
DaVerifierMsg::VerifyShare {commitments, light_share, reply_channel } => {
match verifier.verify(&commitments, &light_share) {
match share_verifier.verify(&commitments, &light_share) {
Ok(()) => {
if let Err(err) = reply_channel.send(Ok(())) {
error!("Error replying verification {err:?}");
@@ -273,7 +467,15 @@ where
},
}
},
}
}
_ = prune_interval.tick() => {
if let Err(err) = Self::prune_pending_txs(
&storage_adapter,
&mempool_trigger,
&mempool_adapter
).await {
error!("Error pruning txs due to {err:?}");
}
}
}
@@ -282,8 +484,15 @@ where
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DaVerifierServiceSettings<BackendSettings, NetworkSettings, StorageSettings> {
pub verifier_settings: BackendSettings,
pub struct DaVerifierServiceSettings<
ShareVerifierSettings,
TxVerifierSettings,
NetworkSettings,
StorageSettings,
> {
pub share_verifier_settings: ShareVerifierSettings,
pub tx_verifier_settings: TxVerifierSettings,
pub network_adapter_settings: NetworkSettings,
pub storage_adapter_settings: StorageSettings,
pub mempool_trigger_settings: MempoolPublishTriggerConfig,
}
nomos-services/data-availability/dispersal/src/adapters/mempool/kzgrs.rs → nomos-services/data-availability/verifier/src/mempool/kzgrs.rs
+16 -27
View File
@@ -1,9 +1,10 @@
use std::{fmt::Debug, marker::PhantomData};
use kzgrs_backend::dispersal::{self, BlobInfo};
use kzgrs_backend::dispersal::{BlobInfo, Index, Metadata};
use nomos_core::{
da::{blob::info::DispersedBlobInfo, BlobId},
header::HeaderId,
mantle::SignedMantleTx,
};
use nomos_da_sampling::backend::DaSamplingServiceBackend;
use nomos_mempool::{
@@ -15,7 +16,7 @@ use overwatch::services::{relay::OutboundRelay, ServiceData};
use serde::{Deserialize, Serialize};
use tokio::sync::oneshot;
use super::{DaMempoolAdapter, DaMempoolAdapterError};
use super::{DaMempoolAdapter, MempoolAdapterError};
type MempoolRelay<Payload, Item, Key> = OutboundRelay<MempoolMsg<HeaderId, Payload, Item, Key>>;
@@ -25,9 +26,6 @@ pub struct KzgrsMempoolAdapter<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
> where
DaPool: MemPool<BlockId = HeaderId>,
@@ -38,7 +36,6 @@ pub struct KzgrsMempoolAdapter<
{
pub mempool_relay: MempoolRelay<DaPoolAdapter::Payload, DaPool::Item, DaPool::Key>,
_phantom: PhantomData<(SamplingBackend, SamplingNetworkAdapter, SamplingStorage)>,
_phantom2: PhantomData<(DaVerifierBackend, DaVerifierNetwork, DaVerifierStorage)>,
}
#[async_trait::async_trait]
@@ -48,9 +45,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
> DaMempoolAdapter
for KzgrsMempoolAdapter<
@@ -59,9 +53,6 @@ impl<
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>
where
@@ -79,11 +70,6 @@ where
SamplingNetworkAdapter:
nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
SamplingStorage: nomos_da_sampling::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierStorage: nomos_da_verifier::storage::DaStorageAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierBackend: nomos_da_verifier::backend::VerifierBackend + Send + Sync + 'static,
DaVerifierBackend::Settings: Clone,
DaVerifierNetwork: nomos_da_verifier::network::NetworkAdapter<RuntimeServiceId> + Send + Sync,
DaVerifierNetwork::Settings: Clone,
{
type MempoolService = DaMempoolService<
DaPoolAdapter,
@@ -91,27 +77,30 @@ where
SamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>;
type BlobId = BlobId;
type Metadata = dispersal::Metadata;
type Tx = SignedMantleTx;
fn new(mempool_relay: OutboundRelay<<Self::MempoolService as ServiceData>::Message>) -> Self {
Self {
mempool_relay,
_phantom: PhantomData,
_phantom2: PhantomData,
}
}
async fn post_blob_id(
// This adapter implementation uses old da mempool. Until chain service
// and indexer are updated to use transaction mempool, old da mempool is
// used for general DA flow testing in integration tests.
async fn post_tx(
&self,
blob_id: Self::BlobId,
metadata: Self::Metadata,
) -> Result<(), DaMempoolAdapterError> {
_tx: Self::Tx,
) -> Result<(), MempoolAdapterError> {
// Metadata will not be used in transaction mempool, it's mocked here for da
// mempool only.
let metadata = Metadata::new([0; 32], Index::from(0));
let (reply_channel, receiver) = oneshot::channel();
self.mempool_relay
.send(MempoolMsg::Add {
@@ -120,8 +109,8 @@ where
reply_channel,
})
.await
.map_err(|(e, _)| DaMempoolAdapterError::from(e))?;
.map_err(|(e, _)| MempoolAdapterError::Other(Box::new(e)))?;
receiver.await?.map_err(DaMempoolAdapterError::Mempool)
receiver.await?.map_err(MempoolAdapterError::Mempool)
}
}
nomos-services/data-availability/verifier/src/mempool/mod.rs
+29
View File
@@ -0,0 +1,29 @@
pub mod kzgrs;
use nomos_mempool::backend::MempoolError;
use overwatch::{
services::{relay::OutboundRelay, ServiceData},
DynError,
};
#[derive(thiserror::Error, Debug)]
pub enum MempoolAdapterError {
#[error("Mempool responded with and error: {0}")]
Mempool(#[from] MempoolError),
#[error("Channel receive error: {0}")]
ChannelRecv(#[from] tokio::sync::oneshot::error::RecvError),
#[error("Other mempool adapter error: {0}")]
Other(DynError),
}
#[async_trait::async_trait]
pub trait DaMempoolAdapter {
type MempoolService: ServiceData;
type BlobId;
type Tx;
fn new(outbound_relay: OutboundRelay<<Self::MempoolService as ServiceData>::Message>) -> Self;
async fn post_tx(&self, blob_id: Self::BlobId, tx: Self::Tx)
-> Result<(), MempoolAdapterError>;
}
nomos-services/data-availability/verifier/src/network/adapters/common.rs
+46 -2
View File
@@ -65,6 +65,7 @@ macro_rules! adapter_for {
type Backend = $DaNetworkBackend<Membership>;
type Settings = ();
type Share = DaShare;
type Tx = SignedMantleTx;
type Membership = Membership;
type Storage = StorageAdapter;
type MembershipAdapter = MembershipServiceAdapter;
@@ -89,7 +90,9 @@ macro_rules! adapter_for {
}
}
async fn share_stream(&self) -> Box<dyn Stream<Item = Self::Share> + Unpin + Send> {
async fn share_stream(
&self,
) -> Box<dyn Stream<Item = ValidationRequest<Self::Share>> + Unpin + Send> {
let (sender, receiver) = tokio::sync::oneshot::channel();
self.network_relay
.send(nomos_da_network_service::DaNetworkMsg::Subscribe {
@@ -102,7 +105,48 @@ macro_rules! adapter_for {
let receiver = receiver.await.expect("Blob stream should be received");
let stream = receiver.filter_map(move |msg| match msg {
$DaNetworkEvent::Verifying(blob) => Some(*blob),
$DaNetworkEvent::Verifying(verification_event) => match verification_event {
VerificationEvent::Share {
share,
response_sender,
} => Some(ValidationRequest {
item: *share,
sender: response_sender,
}),
VerificationEvent::Tx { .. } => None,
},
_ => None,
});
Box::new(Box::pin(stream))
}
async fn tx_stream(
&self,
) -> Box<dyn Stream<Item = ValidationRequest<(u16, Self::Tx)>> + Unpin + Send> {
let (sender, receiver) = tokio::sync::oneshot::channel();
self.network_relay
.send(nomos_da_network_service::DaNetworkMsg::Subscribe {
kind: $DaNetworksEventKind::Verifying,
sender,
})
.await
.expect("Network backend should be ready");
let receiver = receiver.await.expect("Blob stream should be received");
let stream = receiver.filter_map(move |msg| match msg {
$DaNetworkEvent::Verifying(verification_event) => match verification_event {
VerificationEvent::Tx {
assignations,
tx,
response_sender,
} => Some(ValidationRequest {
item: (assignations, *tx),
sender: response_sender,
}),
VerificationEvent::Share { .. } => None,
},
_ => None,
});
nomos-services/data-availability/verifier/src/network/adapters/executor.rs
+6 -3
View File
@@ -3,11 +3,14 @@ use std::{fmt::Debug, marker::PhantomData};
use futures::Stream;
use kzgrs_backend::common::share::{DaShare, DaSharesCommitments};
use libp2p::PeerId;
use nomos_core::da::BlobId;
use nomos_core::{da::BlobId, mantle::SignedMantleTx};
use nomos_da_network_core::SubnetworkId;
use nomos_da_network_service::{
api::ApiAdapter as ApiAdapterTrait,
backends::libp2p::executor::{DaNetworkEvent, DaNetworkEventKind, DaNetworkExecutorBackend},
backends::libp2p::{
common::VerificationEvent,
executor::{DaNetworkEvent, DaNetworkEventKind, DaNetworkExecutorBackend},
},
membership::{handler::DaMembershipHandler, MembershipAdapter},
NetworkService,
};
@@ -15,6 +18,6 @@ use overwatch::services::{relay::OutboundRelay, ServiceData};
use subnetworks_assignations::MembershipHandler;
use tokio_stream::StreamExt as _;
use crate::network::{adapters::common::adapter_for, NetworkAdapter};
use crate::network::{adapters::common::adapter_for, NetworkAdapter, ValidationRequest};
adapter_for!(DaNetworkExecutorBackend, DaNetworkEventKind, DaNetworkEvent);
nomos-services/data-availability/verifier/src/network/adapters/validator.rs
+6 -3
View File
@@ -3,11 +3,14 @@ use std::{fmt::Debug, marker::PhantomData};
use futures::Stream;
use kzgrs_backend::common::share::{DaShare, DaSharesCommitments};
use libp2p::PeerId;
use nomos_core::da::BlobId;
use nomos_core::{da::BlobId, mantle::SignedMantleTx};
use nomos_da_network_core::SubnetworkId;
use nomos_da_network_service::{
api::ApiAdapter as ApiAdapterTrait,
backends::libp2p::validator::{DaNetworkEvent, DaNetworkEventKind, DaNetworkValidatorBackend},
backends::libp2p::{
common::VerificationEvent,
validator::{DaNetworkEvent, DaNetworkEventKind, DaNetworkValidatorBackend},
},
membership::{handler::DaMembershipHandler, MembershipAdapter},
NetworkService,
};
@@ -15,7 +18,7 @@ use overwatch::services::{relay::OutboundRelay, ServiceData};
use subnetworks_assignations::MembershipHandler;
use tokio_stream::StreamExt as _;
use crate::network::{adapters::common::adapter_for, NetworkAdapter};
use crate::network::{adapters::common::adapter_for, NetworkAdapter, ValidationRequest};
adapter_for!(
DaNetworkValidatorBackend,
nomos-services/data-availability/verifier/src/network/mod.rs
+17 -2
View File
@@ -1,15 +1,25 @@
pub mod adapters;
use futures::Stream;
use nomos_da_network_service::{api::ApiAdapter, backends::NetworkBackend, NetworkService};
use nomos_da_network_service::{
api::ApiAdapter,
backends::{libp2p::common::BroadcastValidationResultSender, NetworkBackend},
NetworkService,
};
use overwatch::services::{relay::OutboundRelay, ServiceData};
use subnetworks_assignations::MembershipHandler;
pub struct ValidationRequest<T> {
pub item: T,
pub sender: BroadcastValidationResultSender,
}
#[async_trait::async_trait]
pub trait NetworkAdapter<RuntimeServiceId> {
type Backend: NetworkBackend<RuntimeServiceId> + Send + 'static;
type Settings;
type Share;
type Tx;
type Membership: MembershipHandler + Clone;
type Storage;
type MembershipAdapter;
@@ -29,5 +39,10 @@ pub trait NetworkAdapter<RuntimeServiceId> {
>,
) -> Self;
async fn share_stream(&self) -> Box<dyn Stream<Item = Self::Share> + Unpin + Send>;
async fn share_stream(
&self,
) -> Box<dyn Stream<Item = ValidationRequest<Self::Share>> + Unpin + Send>;
async fn tx_stream(
&self,
) -> Box<dyn Stream<Item = ValidationRequest<(u16, Self::Tx)>> + Unpin + Send>;
}
nomos-services/data-availability/verifier/src/storage/adapters/rocksdb.rs
+43 -2
View File
@@ -1,7 +1,7 @@
use std::{fmt::Debug, hash::Hash, marker::PhantomData, path::PathBuf};
use futures::try_join;
use nomos_core::da::blob::Share;
use nomos_core::{da::blob::Share, mantle::SignedMantleTx};
use nomos_storage::{
api::da::DaConverter,
backends::{rocksdb::RocksBackend, StorageSerde},
@@ -34,11 +34,12 @@ where
B::LightShare: Send + Sync + 'static,
B::SharesCommitments: Send + Sync + 'static,
S: StorageSerde + Send + Sync + 'static,
Converter: DaConverter<RocksBackend<S>, Share = B> + Send + Sync + 'static,
Converter: DaConverter<RocksBackend<S>, Share = B, Tx = SignedMantleTx> + Send + Sync + 'static,
{
type Backend = RocksBackend<S>;
type Share = B;
type Settings = RocksAdapterSettings;
type Tx = SignedMantleTx;
async fn new(
storage_relay: OutboundRelay<
@@ -99,6 +100,46 @@ where
.transpose()
.map_err(DynError::from)
}
async fn add_tx(
&self,
blob_id: <Self::Share as Share>::BlobId,
assignations: u16,
tx: Self::Tx,
) -> Result<(), DynError> {
let store_tx_msg =
StorageMsg::store_tx_request::<Converter>(blob_id.clone(), assignations, tx)?;
self.storage_relay
.send(store_tx_msg)
.await
.map_err(|(e, _)| DynError::from(e))?;
Ok(())
}
async fn get_tx(
&self,
blob_id: <Self::Share as Share>::BlobId,
) -> Result<Option<(u16, Self::Tx)>, DynError> {
let (reply_channel, reply_rx) = tokio::sync::oneshot::channel();
self.storage_relay
.send(StorageMsg::get_tx_request::<Converter>(
blob_id.clone(),
reply_channel,
)?)
.await
.expect("Failed to send request to storage relay");
reply_rx
.await
.map_err(DynError::from)?
.map(|(assignations, data)| {
Converter::tx_from_storage(data).map(|tx| (assignations, tx))
})
.transpose()
.map_err(DynError::from)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
nomos-services/data-availability/verifier/src/storage/mod.rs
+13
View File
@@ -12,6 +12,7 @@ pub trait DaStorageAdapter<RuntimeServiceId> {
type Backend: StorageBackend + Send + Sync + 'static;
type Settings: Clone;
type Share: Share + Clone;
type Tx;
async fn new(
storage_relay: OutboundRelay<
@@ -32,4 +33,16 @@ pub trait DaStorageAdapter<RuntimeServiceId> {
blob_id: <Self::Share as Share>::BlobId,
share_idx: <Self::Share as Share>::ShareIndex,
) -> Result<Option<<Self::Share as Share>::LightShare>, DynError>;
async fn add_tx(
&self,
blob_id: <Self::Share as Share>::BlobId,
assignations: u16,
tx: Self::Tx,
) -> Result<(), DynError>;
async fn get_tx(
&self,
blob_id: <Self::Share as Share>::BlobId,
) -> Result<Option<(u16, Self::Tx)>, DynError>;
}
nomos-services/mempool/Cargo.toml
-1
View File
@@ -22,7 +22,6 @@ linked-hash-map = { version = "0.5.6", optional = true, features = ["serde
nomos-core = { workspace = true }
nomos-da-network-core = { workspace = true }
nomos-da-sampling = { workspace = true, features = ["rocksdb-backend"] }
nomos-da-verifier = { workspace = true, features = ["rocksdb-backend"] }
nomos-network = { workspace = true }
overwatch = { workspace = true }
rand = { workspace = true }
nomos-services/mempool/src/da/service.rs
+25 -50
View File
@@ -10,7 +10,7 @@ use std::{
time::Duration,
};
use futures::StreamExt as _;
use futures::{stream::FuturesUnordered, StreamExt as _};
use nomos_da_sampling::{
backend::DaSamplingServiceBackend, storage::DaStorageAdapter, DaSamplingService,
DaSamplingServiceMsg,
@@ -43,9 +43,6 @@ pub type DaMempoolService<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
> = GenericDaMempoolService<
Pool,
@@ -64,9 +61,6 @@ pub type DaMempoolService<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>;
@@ -79,9 +73,6 @@ pub struct GenericDaMempoolService<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
> where
Pool: RecoverableMempool,
@@ -90,19 +81,12 @@ pub struct GenericDaMempoolService<
{
pool: Pool,
service_resources_handle: OpaqueServiceResourcesHandle<Self, RuntimeServiceId>,
#[expect(
clippy::type_complexity,
reason = "There is nothing we can do about this, at the moment."
)]
_phantom: PhantomData<(
NetworkAdapter,
RecoveryBackend,
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
)>,
}
@@ -113,9 +97,6 @@ impl<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>
GenericDaMempoolService<
@@ -125,9 +106,6 @@ impl<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>
where
@@ -156,9 +134,6 @@ impl<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
> ServiceData
for GenericDaMempoolService<
@@ -168,9 +143,6 @@ impl<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>
where
@@ -192,9 +164,6 @@ impl<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
> ServiceCore<RuntimeServiceId>
for GenericDaMempoolService<
@@ -204,9 +173,6 @@ impl<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>
where
@@ -225,9 +191,6 @@ where
DaSamplingBackend::BlobId: Send + 'static,
DaSamplingNetwork: nomos_da_sampling::network::NetworkAdapter<RuntimeServiceId> + Send,
DaSamplingStorage: DaStorageAdapter<RuntimeServiceId> + Send,
DaVerifierBackend: Send,
DaVerifierNetwork: Send,
DaVerifierStorage: Send,
RuntimeServiceId: Debug
+ Sync
+ Display
@@ -240,9 +203,6 @@ where
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>,
>,
@@ -278,7 +238,7 @@ where
let sampling_relay = self
.service_resources_handle
.overwatch_handle
.relay::<DaSamplingService<_, _, _, _, _, _, _>>()
.relay::<DaSamplingService<_, _, _, _>>()
.await
.expect("Relay connection with SamplingService should succeed");
@@ -301,11 +261,19 @@ where
<RuntimeServiceId as AsServiceId<Self>>::SERVICE_ID
);
let mut trigger_sampling_tasks = FuturesUnordered::new();
let trigger_sampling_delay = self
.service_resources_handle
.settings_handle
.notifier()
.get_updated_settings()
.trigger_sampling_delay;
wait_until_services_are_ready!(
&self.service_resources_handle.overwatch_handle,
Some(Duration::from_secs(60)),
NetworkService<_, _>,
DaSamplingService<_, _, _, _, _, _, _>
DaSamplingService<_, _, _, _>
)
.await?;
@@ -316,13 +284,26 @@ where
self.handle_mempool_message(relay_msg, network_service_relay.clone());
}
Some((key, item )) = network_items.next() => {
sampling_relay.send(DaSamplingServiceMsg::TriggerSampling{blob_id: key.clone()}).await.unwrap_or_else(|_| panic!("Sampling trigger message needs to be sent"));
let sampling_relay_clone = sampling_relay.clone();
let blob_id = key.clone();
trigger_sampling_tasks.push(async move {
tokio::time::sleep(trigger_sampling_delay).await;
sampling_relay_clone.send(DaSamplingServiceMsg::TriggerSampling{
blob_id
}).await
});
self.pool.add_item(key, item).unwrap_or_else(|e| {
tracing::debug!("could not add item to the pool due to: {e}");
});
tracing::info!(counter.da_mempool_pending_items = self.pool.pending_item_count());
self.service_resources_handle.state_updater.update(Some(self.pool.save().into()));
}
Some(result) = trigger_sampling_tasks.next() => {
if let Err((e, _)) = result {
tracing::error!("coulnd not trigger sampling due to {e}");
}
},
}
}
}
@@ -335,9 +316,6 @@ impl<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>
GenericDaMempoolService<
@@ -347,9 +325,6 @@ impl<
DaSamplingBackend,
DaSamplingNetwork,
DaSamplingStorage,
DaVerifierBackend,
DaVerifierNetwork,
DaVerifierStorage,
RuntimeServiceId,
>
where
nomos-services/mempool/src/da/settings.rs
+3 -1
View File
@@ -1,4 +1,4 @@
use std::path::PathBuf;
use std::{path::PathBuf, time::Duration};
use serde::{Deserialize, Serialize};
use services_utils::overwatch::recovery::backends::FileBackendSettings;
@@ -12,6 +12,8 @@ pub struct DaMempoolSettings<PoolSettings, NetworkAdapterSettings> {
pub network_adapter: NetworkAdapterSettings,
/// The recovery file path, for the service's [`RecoveryOperator`].
pub recovery_path: PathBuf,
/// Trigger sampling delay.
pub trigger_sampling_delay: Duration,
}
impl<PoolSettings, NetworkAdapterSettings> FileBackendSettings
nomos-services/mempool/src/tx/service.rs
+10 -41
View File
@@ -12,7 +12,6 @@ use std::{
use futures::StreamExt as _;
use nomos_da_sampling::backend::kzgrs::KzgrsSamplingBackend;
use nomos_da_verifier::backend::kzgrs::KzgrsDaVerifier;
use nomos_network::{message::BackendNetworkMsg, NetworkService};
use overwatch::{
services::{relay::OutboundRelay, AsServiceId, ServiceCore, ServiceData},
@@ -34,70 +33,40 @@ use crate::{
MempoolMetrics, MempoolMsg,
};
pub type DaSamplingService<
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
> = nomos_da_sampling::DaSamplingService<
KzgrsSamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
KzgrsDaVerifier,
VerifierNetworkAdapter,
VerifierStorage,
RuntimeServiceId,
>;
pub type DaSamplingService<SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId> =
nomos_da_sampling::DaSamplingService<
KzgrsSamplingBackend,
SamplingNetworkAdapter,
SamplingStorage,
RuntimeServiceId,
>;
/// A tx mempool service that uses a [`JsonFileBackend`] as a recovery
/// mechanism.
pub type TxMempoolService<
MempoolNetworkAdapter,
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
Pool,
RuntimeServiceId,
> = GenericTxMempoolService<
Pool,
MempoolNetworkAdapter,
SignedTxProcessor<
DaSamplingService<
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>,
>,
SignedTxProcessor<DaSamplingService<SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>>,
JsonFileBackend<
TxMempoolState<
<Pool as RecoverableMempool>::RecoveryState,
<Pool as MemPool>::Settings,
<MempoolNetworkAdapter as NetworkAdapterTrait<RuntimeServiceId>>::Settings,
<SignedTxProcessor<
DaSamplingService<
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>,
DaSamplingService<SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>,
> as PayloadProcessor>::Settings,
>,
TxMempoolSettings<
<Pool as MemPool>::Settings,
<MempoolNetworkAdapter as NetworkAdapterTrait<RuntimeServiceId>>::Settings,
<SignedTxProcessor<
DaSamplingService<
SamplingNetworkAdapter,
VerifierNetworkAdapter,
SamplingStorage,
VerifierStorage,
RuntimeServiceId,
>,
DaSamplingService<SamplingNetworkAdapter, SamplingStorage, RuntimeServiceId>,
> as PayloadProcessor>::Settings,
>,
>,
nomos-services/storage/src/api/backend/rocksdb/da.rs
+51
View File
@@ -22,6 +22,7 @@ pub const DA_SHARED_COMMITMENTS_PREFIX: &str = concat!("da/verified/", "sc");
pub const DA_SHARE_PREFIX: &str = concat!("da/verified/", "bl");
pub const DA_ASSIGNATIONS_PREFIX: &str = concat!("da/membership/", "as");
pub const DA_ADDRESSBOOK_PREFIX: &str = concat!("da/membership/", "ab");
pub const DA_TX_PREFIX: &str = concat!("da/verified/", "tx");
#[async_trait]
impl<SerdeOp: StorageSerde + Send + Sync + 'static> StorageDaApi for RocksBackend<SerdeOp> {
@@ -29,6 +30,7 @@ impl<SerdeOp: StorageSerde + Send + Sync + 'static> StorageDaApi for RocksBacken
type BlobId = BlobId;
type Share = Bytes;
type Commitments = Bytes;
type Tx = Bytes;
type ShareIndex = [u8; 2];
type NetworkId = u16;
type Id = PeerId;
@@ -239,4 +241,53 @@ impl<SerdeOp: StorageSerde + Send + Sync + 'static> StorageDaApi for RocksBacken
},
)
}
async fn store_tx(
&mut self,
blob_id: Self::BlobId,
assignations: u16,
tx: Self::Tx,
) -> Result<(), Self::Error> {
let tx_key = key_bytes(DA_TX_PREFIX, blob_id.as_ref());
let serialized_tx_body = SerdeOp::serialize(tx);
let mut serialized_tx = Vec::with_capacity(2 + serialized_tx_body.len());
serialized_tx.extend_from_slice(&assignations.to_be_bytes());
serialized_tx.extend_from_slice(&serialized_tx_body);
self.store(tx_key, serialized_tx.into()).await
}
async fn get_tx(
&mut self,
blob_id: Self::BlobId,
) -> Result<Option<(u16, Self::Tx)>, Self::Error> {
let tx_key = key_bytes(DA_TX_PREFIX, blob_id.as_ref());
let storage_bytes = self.load(&tx_key).await?;
let Some(mut assignations) = storage_bytes else {
return Ok(None);
};
let tx_bytes = assignations.split_off(2);
let assignations_arr: [u8; 2] = match assignations[..2].try_into() {
Ok(arr) => arr,
Err(e) => {
error!("Failed to convert assignations: {:?}", e);
return Ok(None);
}
};
let assignations = u16::from_be_bytes(assignations_arr);
let tx = match SerdeOp::deserialize::<Self::Tx>(tx_bytes) {
Ok(tx) => Some((assignations, tx)),
Err(e) => {
error!("Failed to deserialize tx: {:?}", e);
None
}
};
Ok(tx)
}
}
nomos-services/storage/src/api/da/mod.rs
+20
View File
@@ -21,8 +21,11 @@ type ServiceLightShare<Converter, Backend> =
type ServiceSharedCommitments<Converter, Backend> =
<<Converter as DaConverter<Backend>>::Share as Share>::SharesCommitments;
type ServiceTx<Converter, Backend> = <Converter as DaConverter<Backend>>::Tx;
pub trait DaConverter<Backend: StorageDaApi> {
type Share: Share;
type Tx;
type Error: Error + Send + Sync + 'static;
fn blob_id_to_storage(
@@ -56,6 +59,10 @@ pub trait DaConverter<Backend: StorageDaApi> {
fn commitments_from_storage(
backend_commitments: Backend::Commitments,
) -> Result<ServiceSharedCommitments<Self, Backend>, Self::Error>;
fn tx_to_storage(service_tx: ServiceTx<Self, Backend>) -> Result<Backend::Tx, Self::Error>;
fn tx_from_storage(backend_tx: Backend::Tx) -> Result<ServiceTx<Self, Backend>, Self::Error>;
}
#[async_trait]
@@ -67,6 +74,7 @@ pub trait StorageDaApi {
type ShareIndex: Send + Sync;
type Id: Send + Sync;
type NetworkId: Send + Sync;
type Tx: Send + Sync;
async fn get_light_share(
&mut self,
@@ -119,4 +127,16 @@ pub trait StorageDaApi {
) -> Result<(), Self::Error>;
async fn get_address(&mut self, id: Self::Id) -> Result<Option<Multiaddr>, Self::Error>;
async fn store_tx(
&mut self,
blob_id: Self::BlobId,
assignations: u16,
tx: Self::Tx,
) -> Result<(), Self::Error>;
async fn get_tx(
&mut self,
blob_id: Self::BlobId,
) -> Result<Option<(u16, Self::Tx)>, Self::Error>;
}
nomos-services/storage/src/api/da/requests.rs
+78
View File
@@ -5,6 +5,7 @@ use nomos_core::block::BlockNumber;
use overwatch::DynError;
use tokio::sync::oneshot::Sender;
use super::ServiceTx;
use crate::{
api::{
da::{
@@ -66,6 +67,15 @@ pub enum DaApiRequest<Backend: StorageBackend> {
id: Backend::Id,
response_tx: Sender<Option<Multiaddr>>,
},
StoreTx {
blob_id: <Backend as StorageDaApi>::BlobId,
tx: <Backend as StorageDaApi>::Tx,
assignations: u16,
},
GetTx {
blob_id: <Backend as StorageDaApi>::BlobId,
response_tx: Sender<Option<(u16, <Backend as StorageDaApi>::Tx)>>,
},
}
impl<Backend> StorageOperation<Backend> for DaApiRequest<Backend>
@@ -112,6 +122,15 @@ where
Self::GetAddress { id, response_tx } => {
handle_get_address(backend, id, response_tx).await
}
Self::StoreTx {
blob_id,
assignations,
tx,
} => handle_store_tx(backend, blob_id, assignations, tx).await,
Self::GetTx {
blob_id,
response_tx,
} => handle_get_tx(backend, blob_id, response_tx).await,
}
}
}
@@ -224,6 +243,18 @@ async fn handle_store_assignations<Backend: StorageBackend>(
.map_err(|e| StorageServiceError::BackendError(e.into()))
}
async fn handle_store_tx<Backend: StorageBackend>(
backend: &mut Backend,
blob_id: Backend::BlobId,
assignations: u16,
tx: Backend::Tx,
) -> Result<(), StorageServiceError> {
backend
.store_tx(blob_id, assignations, tx)
.await
.map_err(|e| StorageServiceError::BackendError(e.into()))
}
async fn handle_get_assignations<Backend: StorageBackend>(
backend: &mut Backend,
block_number: BlockNumber,
@@ -270,6 +301,24 @@ async fn handle_get_address<Backend: StorageBackend>(
Ok(())
}
async fn handle_get_tx<Backend: StorageBackend>(
backend: &mut Backend,
blob_id: <Backend as StorageDaApi>::BlobId,
response_tx: Sender<Option<(u16, <Backend as StorageDaApi>::Tx)>>,
) -> Result<(), StorageServiceError> {
let result = backend
.get_tx(blob_id)
.await
.map_err(|e| StorageServiceError::BackendError(e.into()))?;
if response_tx.send(result).is_err() {
return Err(StorageServiceError::ReplyError {
message: "Failed to send reply for get tx for share".to_owned(),
});
}
Ok(())
}
impl<Backend: StorageBackend> StorageMsg<Backend> {
pub fn get_light_share_request<Converter: DaConverter<Backend>>(
blob_id: ServiceBlobId<Converter, Backend>,
@@ -403,4 +452,33 @@ impl<Backend: StorageBackend> StorageMsg<Backend> {
request: StorageApiRequest::Da(DaApiRequest::GetAddress { id, response_tx }),
}
}
pub fn store_tx_request<Converter: DaConverter<Backend>>(
blob_id: ServiceBlobId<Converter, Backend>,
assignations: u16,
tx: ServiceTx<Converter, Backend>,
) -> Result<Self, DynError> {
let blob_id = Converter::blob_id_to_storage(blob_id).map_err(Into::<DynError>::into)?;
let tx = Converter::tx_to_storage(tx)?;
Ok(Self::Api {
request: StorageApiRequest::Da(DaApiRequest::StoreTx {
blob_id,
tx,
assignations,
}),
})
}
pub fn get_tx_request<Converter: DaConverter<Backend>>(
blob_id: ServiceBlobId<Converter, Backend>,
response_tx: Sender<Option<(u16, <Backend as StorageDaApi>::Tx)>>,
) -> Result<Self, DynError> {
let blob_id = Converter::blob_id_to_storage(blob_id).map_err(Into::<DynError>::into)?;
Ok(Self::Api {
request: StorageApiRequest::Da(DaApiRequest::GetTx {
blob_id,
response_tx,
}),
})
}
}
nomos-services/storage/src/backends/mock.rs
+17
View File
@@ -158,6 +158,7 @@ impl<SerdeOp: StorageSerde + Send + Sync + 'static> StorageDaApi for MockStorage
type BlobId = [u8; 32];
type Share = Bytes;
type Commitments = Bytes;
type Tx = ();
type ShareIndex = [u8; 2];
type Id = PeerId;
type NetworkId = u16;
@@ -233,6 +234,22 @@ impl<SerdeOp: StorageSerde + Send + Sync + 'static> StorageDaApi for MockStorage
async fn get_address(&mut self, _id: Self::Id) -> Result<Option<Multiaddr>, Self::Error> {
unimplemented!()
}
async fn get_tx(
&mut self,
_blob_id: Self::BlobId,
) -> Result<Option<(u16, Self::Tx)>, Self::Error> {
unimplemented!()
}
async fn store_tx(
&mut self,
_blob_id: Self::BlobId,
_assignations: u16,
_tx: Self::Tx,
) -> Result<(), Self::Error> {
unimplemented!()
}
}
#[async_trait]
testnet/cfgsync/Cargo.toml
-1
View File
@@ -12,7 +12,6 @@ axum = { version = "0.6" }
clap = { version = "4", features = ["derive"] }
nomos-blend-scheduling = { workspace = true }
nomos-core = { workspace = true }
nomos-da-dispersal = { workspace = true }
nomos-da-network-core = { workspace = true }
nomos-executor = { workspace = true }
nomos-libp2p = { workspace = true }
testnet/cfgsync/src/config.rs
-2
View File
@@ -287,7 +287,6 @@ pub fn create_membership_configs(ids: &[[u8; 32]], hosts: &[Host]) -> Vec<Genera
mod cfgsync_tests {
use std::{net::Ipv4Addr, num::NonZero, str::FromStr as _, time::Duration};
use nomos_da_dispersal::backend::kzgrs::MempoolPublishStrategy;
use nomos_da_network_core::swarm::{
DAConnectionMonitorSettings, DAConnectionPolicySettings, ReplicationConfig,
};
@@ -327,7 +326,6 @@ mod cfgsync_tests {
old_blobs_check_interval: Duration::from_secs(5),
blobs_validity_duration: Duration::from_secs(u64::MAX),
global_params_path: String::new(),
mempool_strategy: MempoolPublishStrategy::Immediately,
policy_settings: DAConnectionPolicySettings::default(),
monitor_settings: DAConnectionMonitorSettings::default(),
balancer_interval: Duration::ZERO,
testnet/cfgsync/src/server.rs
-3
View File
@@ -1,7 +1,6 @@
use std::{fs, net::Ipv4Addr, num::NonZero, path::PathBuf, sync::Arc, time::Duration};
use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::post, Json, Router};
use nomos_da_dispersal::backend::kzgrs::MempoolPublishStrategy;
use nomos_da_network_core::swarm::{
DAConnectionMonitorSettings, DAConnectionPolicySettings, ReplicationConfig,
};
@@ -47,7 +46,6 @@ pub struct CfgSyncConfig {
pub monitor_failure_time_window: Duration,
#[serde_as(as = "MinimalBoundedDuration<0, SECOND>")]
pub balancer_interval: Duration,
pub mempool_publish_strategy: MempoolPublishStrategy,
pub replication_settings: ReplicationConfig,
pub retry_shares_limit: usize,
pub retry_commitments_limit: usize,
@@ -83,7 +81,6 @@ impl CfgSyncConfig {
old_blobs_check_interval: self.old_blobs_check_interval,
blobs_validity_duration: self.blobs_validity_duration,
global_params_path: self.global_params_path.clone(),
mempool_strategy: self.mempool_publish_strategy.clone(),
policy_settings: DAConnectionPolicySettings {
min_dispersal_peers: self.min_dispersal_peers,
min_replication_peers: self.min_replication_peers,
tests/src/common/da.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use reqwest::Url;
use crate::{adjust_timeout, nodes::executor::Executor};
pub const APP_ID: &str = "fd3384e132ad02a56c78f45547ee40038dc79002b90d29ed90e08eee762ae715";
pub const APP_ID: &str = "0000000000000000000000000000000000000000000000000000000000000000";
pub const DA_TESTS_TIMEOUT: u64 = 120;
pub async fn disseminate_with_metadata(
executor: &Executor,
tests/src/nodes/executor.rs
+20 -5
View File
@@ -37,9 +37,13 @@ use nomos_da_network_service::{
},
MembershipResponse, NetworkConfig as DaNetworkConfig,
};
use nomos_da_sampling::{backend::kzgrs::KzgrsSamplingBackendSettings, DaSamplingServiceSettings};
use nomos_da_sampling::{
backend::kzgrs::KzgrsSamplingBackendSettings,
verifier::kzgrs::KzgrsDaVerifierSettings as SamplingVerifierSettings,
DaSamplingServiceSettings,
};
use nomos_da_verifier::{
backend::kzgrs::KzgrsDaVerifierSettings,
backend::{kzgrs::KzgrsDaVerifierSettings, trigger::MempoolPublishTriggerConfig},
storage::adapters::rocksdb::RocksAdapterSettings as VerifierStorageAdapterSettings,
DaVerifierServiceSettings,
};
@@ -374,7 +378,6 @@ pub fn create_executor_config(config: GeneralConfig) -> Config {
shares_retry_limit: config.da_config.retry_shares_limit,
commitments_retry_limit: config.da_config.retry_commitments_limit,
},
refresh_interval: config.da_config.subnets_refresh_interval,
},
num_subnets: config.da_config.num_subnets,
},
@@ -383,6 +386,7 @@ pub fn create_executor_config(config: GeneralConfig) -> Config {
api_port: config.api_config.address.port(),
is_secure: false,
},
subnet_refresh_interval: config.da_config.subnets_refresh_interval,
},
da_indexer: IndexerSettings {
storage: IndexerStorageAdapterSettings {
@@ -390,14 +394,21 @@ pub fn create_executor_config(config: GeneralConfig) -> Config {
},
},
da_verifier: DaVerifierServiceSettings {
verifier_settings: KzgrsDaVerifierSettings {
share_verifier_settings: KzgrsDaVerifierSettings {
global_params_path: config.da_config.global_params_path.clone(),
domain_size: config.da_config.num_subnets as usize,
},
tx_verifier_settings: (),
network_adapter_settings: (),
storage_adapter_settings: VerifierStorageAdapterSettings {
blob_storage_directory: "./".into(),
},
mempool_trigger_settings: MempoolPublishTriggerConfig {
publish_threshold: NonNegativeF64::try_from(0.8).unwrap(),
share_duration: Duration::from_secs(5),
prune_duration: Duration::from_secs(30),
prune_interval: Duration::from_secs(5),
},
},
tracing: config.tracing_config.tracing_settings,
http: nomos_api::ApiServiceSettings {
@@ -414,6 +425,10 @@ pub fn create_executor_config(config: GeneralConfig) -> Config {
old_blobs_check_interval: config.da_config.old_blobs_check_interval,
blobs_validity_duration: config.da_config.blobs_validity_duration,
},
share_verifier_settings: SamplingVerifierSettings {
global_params_path: config.da_config.global_params_path.clone(),
domain_size: config.da_config.num_subnets as usize,
},
},
storage: RocksBackendSettings {
db_path: "./db".into(),
@@ -428,7 +443,6 @@ pub fn create_executor_config(config: GeneralConfig) -> Config {
global_params_path: config.da_config.global_params_path,
},
dispersal_timeout: Duration::from_secs(20),
mempool_strategy: config.da_config.mempool_strategy,
},
},
time: TimeServiceSettings {
@@ -450,6 +464,7 @@ pub fn create_executor_config(config: GeneralConfig) -> Config {
mempool: MempoolConfig {
cl_pool_recovery_path: "./recovery/cl_mempool.json".into(),
da_pool_recovery_path: "./recovery/da_mempool.json".into(),
trigger_sampling_delay: adjust_timeout(Duration::from_secs(5)),
},
membership: config.membership_config.service_settings,
sdp: (),
tests/src/nodes/validator.rs
+21 -5
View File
@@ -30,9 +30,13 @@ use nomos_da_network_service::{
api::http::ApiAdapterSettings, backends::libp2p::common::DaNetworkBackendSettings,
NetworkConfig as DaNetworkConfig,
};
use nomos_da_sampling::{backend::kzgrs::KzgrsSamplingBackendSettings, DaSamplingServiceSettings};
use nomos_da_sampling::{
backend::kzgrs::KzgrsSamplingBackendSettings,
verifier::kzgrs::KzgrsDaVerifierSettings as SamplingVerifierSettings,
DaSamplingServiceSettings,
};
use nomos_da_verifier::{
backend::kzgrs::KzgrsDaVerifierSettings,
backend::{kzgrs::KzgrsDaVerifierSettings, trigger::MempoolPublishTriggerConfig},
storage::adapters::rocksdb::RocksAdapterSettings as VerifierStorageAdapterSettings,
DaVerifierServiceSettings,
};
@@ -409,13 +413,13 @@ pub fn create_validator_config(config: GeneralConfig) -> Config {
shares_retry_limit: config.da_config.retry_shares_limit,
commitments_retry_limit: config.da_config.retry_commitments_limit,
},
refresh_interval: config.da_config.subnets_refresh_interval,
},
membership: config.da_config.membership.clone(),
api_adapter_settings: ApiAdapterSettings {
api_port: config.api_config.address.port(),
is_secure: false,
},
subnet_refresh_interval: config.da_config.subnets_refresh_interval,
},
da_indexer: IndexerSettings {
storage: IndexerStorageAdapterSettings {
@@ -423,14 +427,21 @@ pub fn create_validator_config(config: GeneralConfig) -> Config {
},
},
da_verifier: DaVerifierServiceSettings {
verifier_settings: KzgrsDaVerifierSettings {
global_params_path: config.da_config.global_params_path,
share_verifier_settings: KzgrsDaVerifierSettings {
global_params_path: config.da_config.global_params_path.clone(),
domain_size: config.da_config.num_subnets as usize,
},
tx_verifier_settings: (),
network_adapter_settings: (),
storage_adapter_settings: VerifierStorageAdapterSettings {
blob_storage_directory: "./".into(),
},
mempool_trigger_settings: MempoolPublishTriggerConfig {
publish_threshold: NonNegativeF64::try_from(0.8).unwrap(),
share_duration: Duration::from_secs(5),
prune_duration: Duration::from_secs(30),
prune_interval: Duration::from_secs(5),
},
},
tracing: config.tracing_config.tracing_settings,
http: nomos_api::ApiServiceSettings {
@@ -447,6 +458,10 @@ pub fn create_validator_config(config: GeneralConfig) -> Config {
old_blobs_check_interval: config.da_config.old_blobs_check_interval,
blobs_validity_duration: config.da_config.blobs_validity_duration,
},
share_verifier_settings: SamplingVerifierSettings {
global_params_path: config.da_config.global_params_path,
domain_size: config.da_config.num_subnets as usize,
},
},
storage: RocksBackendSettings {
db_path: "./db".into(),
@@ -473,6 +488,7 @@ pub fn create_validator_config(config: GeneralConfig) -> Config {
mempool: MempoolConfig {
cl_pool_recovery_path: "./recovery/cl_mempool.json".into(),
da_pool_recovery_path: "./recovery/da_mempool.json".into(),
trigger_sampling_delay: adjust_timeout(Duration::from_secs(5)),
},
membership: config.membership_config.service_settings,
sdp: (),
tests/src/tests/da/disperse.rs
+2 -2
View File
@@ -51,6 +51,7 @@ async fn disseminate_and_retrieve() {
assert!(validator_idx_0_blobs.count() == 2);
}
#[ignore = "Reenable after transaction mempool is used"]
#[tokio::test]
async fn disseminate_retrieve_reconstruct() {
const ITERATIONS: usize = 10;
@@ -204,8 +205,7 @@ async fn disseminate_same_data() {
let from = 0u64.to_be_bytes();
let to = 1u64.to_be_bytes();
for i in 0..ITERATIONS {
println!("iteration {i}");
for _ in 0..ITERATIONS {
disseminate_with_metadata(executor, &data, metadata).await;
wait_for_indexed_blob(executor, app_id, from, to, num_subnets).await;
tests/src/topology/configs/da.rs
+1 -10
View File
@@ -2,7 +2,6 @@ use std::{
collections::HashSet, env, path::PathBuf, str::FromStr as _, sync::LazyLock, time::Duration,
};
use nomos_da_dispersal::backend::kzgrs::{MempoolPublishStrategy, SampleSubnetworks};
use nomos_da_network_core::swarm::{
DAConnectionMonitorSettings, DAConnectionPolicySettings, ReplicationConfig,
};
@@ -32,7 +31,6 @@ pub struct DaParams {
pub old_blobs_check_interval: Duration,
pub blobs_validity_duration: Duration,
pub global_params_path: String,
pub mempool_strategy: MempoolPublishStrategy,
pub policy_settings: DAConnectionPolicySettings,
pub monitor_settings: DAConnectionMonitorSettings,
pub balancer_interval: Duration,
@@ -53,11 +51,6 @@ impl Default for DaParams {
old_blobs_check_interval: Duration::from_secs(5),
blobs_validity_duration: Duration::from_secs(60),
global_params_path: GLOBAL_PARAMS_PATH.to_string(),
mempool_strategy: MempoolPublishStrategy::SampleSubnetworks(SampleSubnetworks {
sample_threshold: 2,
timeout: Duration::from_secs(10),
cooldown: Duration::from_millis(100),
}),
policy_settings: DAConnectionPolicySettings {
min_dispersal_peers: 1,
min_replication_peers: 1,
@@ -76,7 +69,7 @@ impl Default for DaParams {
seen_message_cache_size: 1000,
seen_message_ttl: Duration::from_secs(3600),
},
subnets_refresh_interval: Duration::from_secs(5),
subnets_refresh_interval: Duration::from_secs(30),
retry_shares_limit: 1,
retry_commitments_limit: 1,
}
@@ -95,7 +88,6 @@ pub struct GeneralDaConfig {
pub verifier_index: HashSet<u16>,
pub num_samples: u16,
pub num_subnets: u16,
pub mempool_strategy: MempoolPublishStrategy,
pub old_blobs_check_interval: Duration,
pub blobs_validity_duration: Duration,
pub policy_settings: DAConnectionPolicySettings,
@@ -159,7 +151,6 @@ pub fn create_da_configs(
num_subnets: da_params.num_subnets,
old_blobs_check_interval: da_params.old_blobs_check_interval,
blobs_validity_duration: da_params.blobs_validity_duration,
mempool_strategy: da_params.mempool_strategy.clone(),
policy_settings: da_params.policy_settings.clone(),
monitor_settings: da_params.monitor_settings.clone(),
balancer_interval: da_params.balancer_interval,