diff --git a/compose.devnet.run.yml b/compose.devnet.run.yml index 299192b50..2048b5565 100644 --- a/compose.devnet.run.yml +++ b/compose.devnet.run.yml @@ -1,5 +1,8 @@ version: '3.8' +include: + - testnet/compose.tracing.yml + services: explorer: @@ -18,6 +21,7 @@ services: container_name: cfgsync image: ghcr.io/logos-blockchain/logos-blockchain:devnet volumes: + - node-data:/node-data - ./testnet:/etc/logos-blockchain depends_on: - grafana @@ -109,69 +113,6 @@ services: - "18083:18083/tcp" entrypoint: /etc/logos-blockchain/scripts/run_node.sh - prometheus: - container_name: prometheus - image: prom/prometheus:v3.0.1 - volumes: - - ./testnet/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:z - command: - - --config.file=/etc/prometheus/prometheus.yml - - --storage.tsdb.retention.time=7d - - --web.enable-otlp-receiver - ports: - - 0.0.0.0:9090:9090 - restart: on-failure - - grafana: - container_name: grafana - image: grafana/grafana:latest - environment: - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin - - GF_AUTH_DISABLE_LOGIN_FORM=true - - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor traceQLStreaming metricsSummary - env_file: - - ./testnet/monitoring/grafana/plugins.env - volumes: - - ./testnet/monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:z - - ./testnet/monitoring/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:z - - ./testnet/monitoring/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml:z - - ./testnet/monitoring/grafana/dashboards:/var/lib/grafana/dashboards/:z - ports: - - 9091:3000 - restart: on-failure - depends_on: - - prometheus - - loki: - image: grafana/loki:2.9.2 - ports: - - "3100:3100" - command: -config.file=/etc/loki/local-config.yaml - - # Tempo runs as unpriviliged user, volumes need to be chowned before running. - tempo-init: - image: busybox:latest - user: root - entrypoint: - - "chown" - - "10001:10001" - - "/var/tempo" - volumes: - - ./tempo-data:/var/tempo - - tempo: - image: grafana/tempo:latest - container_name: tempo - command: [ "-config.file=/etc/tempo.yaml" ] - volumes: - - ./testnet/monitoring/tempo.yaml:/etc/tempo.yaml:z - - ./tempo-data:/var/tempo - ports: - - "4317:4317" # otlp grpc - depends_on: - - tempo-init - # L2 Sequencer Demo Services l2-nginx: container_name: l2_nginx diff --git a/nodes/node/binary/src/config/tracing/serde/logger.rs b/nodes/node/binary/src/config/tracing/serde/logger.rs index 215abb908..656e72e9a 100644 --- a/nodes/node/binary/src/config/tracing/serde/logger.rs +++ b/nodes/node/binary/src/config/tracing/serde/logger.rs @@ -10,6 +10,7 @@ pub enum Layer { Gelf(GelfConfig), File(FileConfig), Loki(LokiConfig), + Otlp(OtlpConfig), #[default] Stdout, Stderr, @@ -31,6 +32,10 @@ impl From for LoggerLayer { endpoint: config.endpoint, host_identifier: config.host_identifier, }), + Layer::Otlp(config) => Self::Otlp(lb_tracing::logging::otlp::OtlpConfig { + endpoint: config.endpoint, + service_name: config.service_name, + }), Layer::Stdout => Self::Stdout, Layer::Stderr => Self::Stderr, Layer::None => Self::None, @@ -73,3 +78,9 @@ pub struct LokiConfig { pub endpoint: Url, pub host_identifier: String, } + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct OtlpConfig { + pub endpoint: Url, + pub service_name: String, +} diff --git a/testnet/cfgsync.yaml b/testnet/cfgsync.yaml index a0ed5fb8b..faf0f524f 100644 --- a/testnet/cfgsync.yaml +++ b/testnet/cfgsync.yaml @@ -13,7 +13,9 @@ mode: Run # Tracing tracing_settings: - logger: Stdout + logger: !Otlp + endpoint: http://otel:4317 + service_name: init_node tracing: None filter: None metrics: !Otlp diff --git a/testnet/compose.tracing.yml b/testnet/compose.tracing.yml new file mode 100644 index 000000000..785890dc5 --- /dev/null +++ b/testnet/compose.tracing.yml @@ -0,0 +1,67 @@ +services: + + grafana: + container_name: grafana + image: grafana/grafana:latest + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor traceQLStreaming metricsSummary + env_file: + - ./testnet/tracing/grafana/plugins.env + volumes: + - ./testnet/tracing/grafana/grafana.ini:/etc/grafana/grafana.ini:z + - ./testnet/tracing/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:z + - ./testnet/tracing/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml:z + - ./testnet/tracing/grafana/dashboards:/var/lib/grafana/dashboards/:z + ports: + - 9091:3000 + restart: on-failure + depends_on: + - prometheus + - loki + + otel: + image: otel/opentelemetry-collector-contrib:latest + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./testnet/tracing/otel.yaml:/etc/otel-collector-config.yaml + ports: + - "0.0.0.0:3100:4317" + - "14318:4318" + + loki: + image: grafana/loki:latest + ports: + - "23100:3100" + - "24317:4317" + - "24318:4318" + volumes: + - ./testnet/tracing/loki.yaml:/etc/loki/local-config.yaml + command: -config.file=/etc/loki/local-config.yaml + + prometheus: + container_name: prometheus + image: docker.io/prom/prometheus:v3.0.1 + volumes: + - ./testnet/tracing/prometheus.yaml:/etc/prometheus/prometheus.yml:z + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=7d + - --web.enable-otlp-receiver + ports: + - "0.0.0.0:9090:9090" + restart: on-failure + + tempo: + image: grafana/tempo:latest + container_name: tempo + command: [ "-config.file=/etc/tempo.yaml" ] + volumes: + - ./testnet/tracing/tempo.yaml:/etc/tempo.yaml:z + - ./tempo-data:/var/tempo + ports: + - "34317:4317" + depends_on: + - tempo-init diff --git a/testnet/monitoring/grafana/dashboards.yml b/testnet/tracing/grafana/dashboards.yml similarity index 100% rename from testnet/monitoring/grafana/dashboards.yml rename to testnet/tracing/grafana/dashboards.yml diff --git a/testnet/monitoring/grafana/dashboards/testnet_metrics.json b/testnet/tracing/grafana/dashboards/testnet_metrics.json similarity index 100% rename from testnet/monitoring/grafana/dashboards/testnet_metrics.json rename to testnet/tracing/grafana/dashboards/testnet_metrics.json diff --git a/testnet/monitoring/grafana/datasources.yaml b/testnet/tracing/grafana/datasources.yaml similarity index 100% rename from testnet/monitoring/grafana/datasources.yaml rename to testnet/tracing/grafana/datasources.yaml diff --git a/testnet/monitoring/grafana/grafana.ini b/testnet/tracing/grafana/grafana.ini similarity index 100% rename from testnet/monitoring/grafana/grafana.ini rename to testnet/tracing/grafana/grafana.ini diff --git a/testnet/monitoring/grafana/plugins.env b/testnet/tracing/grafana/plugins.env similarity index 100% rename from testnet/monitoring/grafana/plugins.env rename to testnet/tracing/grafana/plugins.env diff --git a/testnet/tracing/loki.yaml b/testnet/tracing/loki.yaml new file mode 100644 index 000000000..1ac85674b --- /dev/null +++ b/testnet/tracing/loki.yaml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + reject_old_samples: true + reject_old_samples_max_age: 168h diff --git a/testnet/tracing/otel.yaml b/testnet/tracing/otel.yaml new file mode 100644 index 000000000..54852db3f --- /dev/null +++ b/testnet/tracing/otel.yaml @@ -0,0 +1,22 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + +exporters: + otlp_http/loki: + endpoint: "http://loki:3100/otlp" + tls: + insecure: true + + debug: + verbosity: detailed + +service: + pipelines: + logs: + receivers: [otlp] + exporters: [otlp_http/loki, debug] diff --git a/testnet/monitoring/prometheus.yml b/testnet/tracing/prometheus.yaml similarity index 100% rename from testnet/monitoring/prometheus.yml rename to testnet/tracing/prometheus.yaml diff --git a/testnet/monitoring/tempo.yaml b/testnet/tracing/tempo.yaml similarity index 100% rename from testnet/monitoring/tempo.yaml rename to testnet/tracing/tempo.yaml