feat(tracing): Otlp http exporter in tracing crate (#3096)

Cargo.lock

@@ -4424,6 +4424,7 @@ dependencies = [
  "logos-blockchain-key-management-system-service",
  "logos-blockchain-libp2p",
  "logos-blockchain-node",
+ "logos-blockchain-tracing",
  "logos-blockchain-utils",
  "num-bigint",
  "rand 0.8.6",

deployment/cfgsync.yaml

@@ -13,9 +13,7 @@ mode: Run
 # Tracing
 tracing_settings:
   logger:
-    otlp:
-      endpoint: "http://otel:4317"
-      service_name: "init_node"
+    otlp: null
     stdout: false
     stderr: false
     file:
@@ -24,8 +22,6 @@ tracing_settings:
     gelf: null
   tracing: None
   filter: None
-  metrics: !Otlp
-    endpoint: "http://otel:4317"
-    host_identifier: "init_node"
+  metrics: None
   console: None
   level: "DEBUG"

deployment/cfgsync/src/config/mod.rs

@@ -271,7 +271,7 @@ fn update_tracing_identifier(
 
     let metrics = match settings.metrics {
         tracing::metrics::Layer::Otlp(mut config) => {
-            config.host_identifier.clone_from(identifier);
+            config.service_name.clone_from(identifier);
             tracing::metrics::Layer::Otlp(config)
         }
         other @ tracing::metrics::Layer::None => other,

nodes/node/binary/src/config/tracing/serde/logger.rs

@@ -56,11 +56,16 @@ impl From<Layers> for LoggerLayerSettings {
             gelf: value
                 .gelf
                 .map(|g| lb_tracing::logging::gelf::GelfConfig { addr: g.addr }),
-            otlp: value.otlp.map(|o| lb_tracing::logging::otlp::OtlpConfig {
-                endpoint: o.endpoint,
-                service_name: o.service_name,
-                authorization_header: o.authorization_header,
-            }),
+            otlp: value
+                .otlp
+                .map(|o| lb_tracing::logging::otlp::OtlpLoggingConfig {
+                    service: lb_tracing::OtlpServiceConfig {
+                        url: o.endpoint,
+                        service_name: o.service_name,
+                        authorization_header: o.authorization_header,
+                        protocol: o.protocol,
+                    },
+                }),
             stdout: value.stdout,
             stderr: value.stderr,
         }
@@ -111,4 +116,5 @@ pub struct OtlpConfig {
     pub service_name: String,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub authorization_header: Option<String>,
+    pub protocol: lb_tracing::OtlpProtocol,
 }

nodes/node/binary/src/config/tracing/serde/metrics.rs

@@ -1,4 +1,4 @@
-use lb_tracing::metrics::otlp::OtlpMetricsConfig;
+use lb_tracing::{OtlpProtocol, OtlpServiceConfig, metrics::otlp::OtlpMetricsConfig};
 use lb_tracing_service::MetricsLayerSettings;
 use serde::{Deserialize, Serialize};
 use url::Url;
@@ -14,9 +14,12 @@ impl From<Layer> for MetricsLayerSettings {
     fn from(value: Layer) -> Self {
         match value {
             Layer::Otlp(config) => Self::Otlp(OtlpMetricsConfig {
-                endpoint: config.endpoint,
-                host_identifier: config.host_identifier,
-                authorization_header: config.authorization_header,
+                service: OtlpServiceConfig {
+                    url: config.endpoint,
+                    service_name: config.service_name,
+                    authorization_header: config.authorization_header,
+                    protocol: config.protocol,
+                },
             }),
             Layer::None => Self::None,
         }
@@ -26,7 +29,8 @@ impl From<Layer> for MetricsLayerSettings {
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct OtlpConfig {
     pub endpoint: Url,
-    pub host_identifier: String,
+    pub service_name: String,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub authorization_header: Option<String>,
+    pub protocol: OtlpProtocol,
 }

nodes/node/binary/src/config/tracing/serde/tracing.rs

@@ -1,4 +1,4 @@
-use lb_tracing::tracing::otlp::OtlpTracingConfig;
+use lb_tracing::{OtlpProtocol, OtlpServiceConfig, tracing::otlp::OtlpTracingConfig};
 use lb_tracing_service::TracingLayerSettings;
 use serde::{Deserialize, Serialize};
 use url::Url;
@@ -14,10 +14,13 @@ impl From<Layer> for TracingLayerSettings {
     fn from(value: Layer) -> Self {
         match value {
             Layer::Otlp(config) => Self::Otlp(OtlpTracingConfig {
-                endpoint: config.endpoint,
+                service: OtlpServiceConfig {
+                    url: config.endpoint,
+                    service_name: config.service_name,
+                    authorization_header: config.authorization_header,
+                    protocol: config.protocol,
+                },
                 sample_ratio: config.sample_ratio,
-                service_name: config.service_name,
-                authorization_header: config.authorization_header,
             }),
             Layer::None => Self::None,
         }
@@ -32,6 +35,7 @@ pub struct OtlpConfig {
     pub service_name: String,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub authorization_header: Option<String>,
+    pub protocol: OtlpProtocol,
 }
 
 const fn default_sample_ratio() -> f64 {

services/tracing/src/lib.rs

@@ -13,7 +13,7 @@ use lb_tracing::{
         gelf::{GelfConfig, create_gelf_layer},
         local::{AppenderType, FileConfig, create_file_layer, create_writer_layer},
         loki::{LokiConfig, create_loki_layer},
-        otlp::{OtlpConfig, create_otlp_layer},
+        otlp::{OtlpLoggingConfig, create_otlp_layer},
     },
     metrics::otlp::{OtlpMetricsConfig, create_otlp_metrics_layer},
     tracing::otlp::{OtlpTracingConfig, create_otlp_tracing_layer},
@@ -164,7 +164,7 @@ pub enum LoggerLayer {
     Gelf(GelfConfig),
     File(FileConfig),
     Loki(LokiConfig),
-    Otlp(OtlpConfig),
+    Otlp(OtlpLoggingConfig),
     Stdout,
     Stderr,
     #[serde(skip)]
@@ -178,7 +178,7 @@ pub struct LoggerLayerSettings {
     pub file: Option<FileConfig>,
     pub loki: Option<LokiConfig>,
     pub gelf: Option<GelfConfig>,
-    pub otlp: Option<OtlpConfig>,
+    pub otlp: Option<OtlpLoggingConfig>,
     pub stdout: bool,
     pub stderr: bool,
 }

tools/config/Cargo.toml

@@ -24,6 +24,7 @@ lb-groth16                       = { workspace = true }
 lb-key-management-system-service = { workspace = true }
 lb-libp2p                        = { workspace = true }
 lb-node                          = { workspace = true }
+lb-tracing                       = { workspace = true }
 lb-utils                         = { workspace = true }
 num-bigint                       = { workspace = true }
 rand                             = { workspace = true }

tools/config/src/tracing.rs

@@ -25,6 +25,7 @@ impl GeneralTracingConfig {
                         endpoint: LOKI_ENDPOINT.try_into().unwrap(),
                         service_name: host_identifier.clone(),
                         authorization_header: None,
+                        protocol: lb_tracing::OtlpProtocol::Grpc,
                     }),
                     stdout: true,
                     file: None,
@@ -37,6 +38,7 @@ impl GeneralTracingConfig {
                     sample_ratio: DEBUG_TRACING_SAMPLE_RATIO,
                     service_name: host_identifier.clone(),
                     authorization_header: None,
+                    protocol: lb_tracing::OtlpProtocol::Grpc,
                 }),
                 filter: tracing::filter::Layer::Env(tracing::filter::EnvConfig {
                     filters: HashMap::from([
@@ -46,8 +48,9 @@ impl GeneralTracingConfig {
                 }),
                 metrics: tracing::metrics::Layer::Otlp(tracing::metrics::OtlpConfig {
                     endpoint: OTLP_METRICS_ENDPOINT.try_into().unwrap(),
-                    host_identifier,
+                    service_name: host_identifier,
                     authorization_header: None,
+                    protocol: lb_tracing::OtlpProtocol::Grpc,
                 }),
                 console: tracing::console::Layer::None,
                 level: tracing::Level::DEBUG,

tracing/Cargo.toml

@@ -31,7 +31,7 @@ opentelemetry-otlp = { features = [
   "http-proto",
   "logs",
   "metrics",
-  "reqwest-client",
+  "reqwest-blocking-client",
 ], workspace = true }
 opentelemetry-semantic-conventions = { workspace = true }
 opentelemetry_sdk = { features = ["logs", "rt-tokio"], workspace = true }

tracing/src/lib.rs

@@ -5,6 +5,8 @@ pub mod metrics;
 pub mod tracing;
 
 pub use opentelemetry;
+use serde::{Deserialize, Serialize};
+use url::Url;
 
 #[macro_export]
 macro_rules! increase_counter_u64 {
@@ -59,3 +61,18 @@ macro_rules! metric_histogram_f64 {
         $crate::metrics::emit::histogram_f64(stringify!($name), $value, attributes);
     }};
 }
+
+#[derive(Clone, Debug, Serialize, Deserialize, Default)]
+pub enum OtlpProtocol {
+    #[default]
+    Grpc,
+    Http,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct OtlpServiceConfig {
+    pub url: Url,
+    pub authorization_header: Option<String>,
+    pub protocol: OtlpProtocol,
+    pub service_name: String,
+}

tracing/src/logging/otlp.rs

@@ -1,42 +1,37 @@
-use std::error::Error;
+use std::{collections::HashMap, error::Error};
 
 use opentelemetry::KeyValue;
 use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
-use opentelemetry_otlp::{WithExportConfig as _, WithTonicConfig as _};
+use opentelemetry_otlp::{WithExportConfig as _, WithHttpConfig as _, WithTonicConfig as _};
 use opentelemetry_sdk::{Resource, logs::SdkLoggerProvider};
 use opentelemetry_semantic_conventions::resource::SERVICE_NAME;
 use serde::{Deserialize, Serialize};
 use tonic::metadata::MetadataMap;
-use url::Url;
+
+use crate::{OtlpProtocol, OtlpServiceConfig};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct OtlpConfig {
-    pub endpoint: Url,
-    pub service_name: String,
-    pub authorization_header: Option<String>,
+pub struct OtlpLoggingConfig {
+    #[serde(flatten)]
+    pub service: OtlpServiceConfig,
 }
 
 pub fn create_otlp_layer(
-    config: OtlpConfig,
+    config: OtlpLoggingConfig,
 ) -> Result<
     OpenTelemetryTracingBridge<SdkLoggerProvider, opentelemetry_sdk::logs::SdkLogger>,
     Box<dyn Error + Send + Sync>,
 > {
     let resource = Resource::builder()
-        .with_attributes(vec![KeyValue::new(SERVICE_NAME, config.service_name)])
+        .with_attributes(vec![KeyValue::new(
+            SERVICE_NAME,
+            config.service.service_name.clone(),
+        )])
         .build();
 
-    let exporter = {
-        let mut exporter = opentelemetry_otlp::LogExporter::builder()
-            .with_tonic()
-            .with_endpoint(config.endpoint.to_string());
-        if let Some(auth_header) = config.authorization_header {
-            let mut metadata = MetadataMap::new();
-            metadata.insert("authorization", auth_header.parse()?);
-            exporter = exporter.with_metadata(metadata);
-        }
-
-        exporter.build()?
+    let exporter = match config.service.protocol {
+        OtlpProtocol::Grpc => build_grpc_exporter(config)?,
+        OtlpProtocol::Http => build_http_exporter(config)?,
     };
 
     let logger_provider = SdkLoggerProvider::builder()
@@ -46,3 +41,33 @@ pub fn create_otlp_layer(
 
     Ok(OpenTelemetryTracingBridge::new(&logger_provider))
 }
+
+fn build_grpc_exporter(
+    config: OtlpLoggingConfig,
+) -> Result<opentelemetry_otlp::LogExporter, Box<dyn Error + Send + Sync>> {
+    let mut builder = opentelemetry_otlp::LogExporter::builder()
+        .with_tonic()
+        .with_endpoint(config.service.url.to_string());
+
+    if let Some(auth) = config.service.authorization_header {
+        let mut metadata = MetadataMap::new();
+        metadata.insert("authorization", auth.parse()?);
+        builder = builder.with_metadata(metadata);
+    }
+
+    Ok(builder.build()?)
+}
+
+fn build_http_exporter(
+    config: OtlpLoggingConfig,
+) -> Result<opentelemetry_otlp::LogExporter, Box<dyn Error + Send + Sync>> {
+    let mut builder = opentelemetry_otlp::LogExporter::builder()
+        .with_http()
+        .with_endpoint(config.service.url.to_string());
+
+    if let Some(auth) = config.service.authorization_header {
+        builder = builder.with_headers(HashMap::from([("authorization".to_owned(), auth)]));
+    }
+
+    Ok(builder.build()?)
+}

tracing/src/metrics/otlp.rs

@@ -1,22 +1,20 @@
-use std::error::Error;
+use std::{collections::HashMap, error::Error};
 
 use opentelemetry::{KeyValue, global};
-use opentelemetry_otlp::{WithExportConfig as _, WithTonicConfig as _};
+use opentelemetry_otlp::{WithExportConfig as _, WithHttpConfig as _, WithTonicConfig as _};
 use opentelemetry_sdk::Resource;
 use serde::{Deserialize, Serialize};
 use tonic::metadata::MetadataMap;
 use tracing::Subscriber;
 use tracing_opentelemetry::MetricsLayer;
 use tracing_subscriber::registry::LookupSpan;
-use url::Url;
 
-use crate::metrics::emit::reset_cached_instruments;
+use crate::{OtlpProtocol, OtlpServiceConfig, metrics::emit::reset_cached_instruments};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct OtlpMetricsConfig {
-    pub endpoint: Url,
-    pub host_identifier: String,
-    pub authorization_header: Option<String>,
+    #[serde(flatten)]
+    pub service: OtlpServiceConfig,
 }
 
 pub fn create_otlp_metrics_layer<S>(
@@ -31,21 +29,13 @@ where
     let resource = Resource::builder_empty()
         .with_attributes(vec![KeyValue::new(
             opentelemetry_semantic_conventions::resource::SERVICE_NAME,
-            config.host_identifier,
+            config.service.service_name.clone(),
         )])
         .build();
 
-    let exporter = {
-        let mut exporter = opentelemetry_otlp::MetricExporter::builder()
-            .with_tonic()
-            .with_endpoint(config.endpoint.to_string());
-        if let Some(auth_header) = config.authorization_header {
-            let mut metadata = MetadataMap::new();
-            metadata.insert("authorization", auth_header.parse()?);
-            exporter = exporter.with_metadata(metadata);
-        }
-
-        exporter.build()?
+    let exporter = match config.service.protocol {
+        OtlpProtocol::Grpc => build_grpc_exporter(config)?,
+        OtlpProtocol::Http => build_http_exporter(config)?,
     };
 
     let meter_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()
@@ -59,3 +49,33 @@ where
     reset_cached_instruments();
     Ok(MetricsLayer::new(meter_provider))
 }
+
+fn build_grpc_exporter(
+    config: OtlpMetricsConfig,
+) -> Result<opentelemetry_otlp::MetricExporter, Box<dyn Error + Send + Sync>> {
+    let mut builder = opentelemetry_otlp::MetricExporter::builder()
+        .with_tonic()
+        .with_endpoint(config.service.url.to_string());
+
+    if let Some(auth) = config.service.authorization_header {
+        let mut metadata = MetadataMap::new();
+        metadata.insert("authorization", auth.parse()?);
+        builder = builder.with_metadata(metadata);
+    }
+
+    Ok(builder.build()?)
+}
+
+fn build_http_exporter(
+    config: OtlpMetricsConfig,
+) -> Result<opentelemetry_otlp::MetricExporter, Box<dyn Error + Send + Sync>> {
+    let mut builder = opentelemetry_otlp::MetricExporter::builder()
+        .with_http()
+        .with_endpoint(config.service.url.to_string());
+
+    if let Some(auth) = config.service.authorization_header {
+        builder = builder.with_headers(HashMap::from([("authorization".to_owned(), auth)]));
+    }
+
+    Ok(builder.build()?)
+}

tracing/src/tracing/otlp.rs

@@ -1,7 +1,7 @@
-use std::error::Error;
+use std::{collections::HashMap, error::Error};
 
 use opentelemetry::{KeyValue, global, trace::TracerProvider as _};
-use opentelemetry_otlp::{WithExportConfig as _, WithTonicConfig as _};
+use opentelemetry_otlp::{WithExportConfig as _, WithHttpConfig as _, WithTonicConfig as _};
 use opentelemetry_sdk::{
     Resource,
     propagation::TraceContextPropagator,
@@ -13,14 +13,14 @@ use tonic::metadata::MetadataMap;
 use tracing::Subscriber;
 use tracing_opentelemetry::OpenTelemetryLayer;
 use tracing_subscriber::registry::LookupSpan;
-use url::Url;
+
+use crate::{OtlpProtocol, OtlpServiceConfig};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct OtlpTracingConfig {
-    pub endpoint: Url,
+    #[serde(flatten)]
+    pub service: OtlpServiceConfig,
     pub sample_ratio: f64,
-    pub service_name: String,
-    pub authorization_header: Option<String>,
 }
 
 pub fn create_otlp_tracing_layer<S>(
@@ -30,26 +30,23 @@ where
     S: Subscriber + for<'span> LookupSpan<'span>,
 {
     let resource = Resource::builder()
-        .with_attributes(vec![KeyValue::new(SERVICE_NAME, config.service_name)])
+        .with_attributes(vec![KeyValue::new(
+            SERVICE_NAME,
+            config.service.service_name.clone(),
+        )])
         .build();
 
-    let exporter = {
-        let mut exporter = opentelemetry_otlp::SpanExporter::builder()
-            .with_tonic()
-            .with_endpoint(config.endpoint.to_string());
-        if let Some(auth_header) = config.authorization_header {
-            let mut metadata = MetadataMap::new();
-            metadata.insert("authorization", auth_header.parse()?);
-            exporter = exporter.with_metadata(metadata);
-        }
+    let sample_ratio = config.sample_ratio;
 
-        exporter.build()?
+    let exporter = match config.service.protocol {
+        OtlpProtocol::Grpc => build_grpc_exporter(config)?,
+        OtlpProtocol::Http => build_http_exporter(config)?,
     };
 
     let tracer_provider = SdkTracerProvider::builder()
         .with_resource(resource)
         .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
-            config.sample_ratio,
+            sample_ratio,
         ))))
         .with_batch_exporter(exporter)
         .build();
@@ -61,3 +58,33 @@ where
 
     Ok(OpenTelemetryLayer::new(tracer))
 }
+
+fn build_grpc_exporter(
+    config: OtlpTracingConfig,
+) -> Result<opentelemetry_otlp::SpanExporter, Box<dyn Error + Send + Sync>> {
+    let mut builder = opentelemetry_otlp::SpanExporter::builder()
+        .with_tonic()
+        .with_endpoint(config.service.url.to_string());
+
+    if let Some(auth) = config.service.authorization_header {
+        let mut metadata = MetadataMap::new();
+        metadata.insert("authorization", auth.parse()?);
+        builder = builder.with_metadata(metadata);
+    }
+
+    Ok(builder.build()?)
+}
+
+fn build_http_exporter(
+    config: OtlpTracingConfig,
+) -> Result<opentelemetry_otlp::SpanExporter, Box<dyn Error + Send + Sync>> {
+    let mut builder = opentelemetry_otlp::SpanExporter::builder()
+        .with_http()
+        .with_endpoint(config.service.url.to_string());
+
+    if let Some(auth) = config.service.authorization_header {
+        builder = builder.with_headers(HashMap::from([("authorization".to_owned(), auth)]));
+    }
+
+    Ok(builder.build()?)
+}