WIP

docker-compose.yml

@@ -195,26 +195,27 @@ services:
       retries: 5
       start_period: 80s
 
-  # postgres-exporter:
+  postgres-exporter:
-  #   # Service aimed to scrape information from Postgres and post it to Prometeus
+    # Service aimed to scrape information from Postgres and post it to Prometeus
-  #   image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0
+    image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0
-  #   restart: on-failure:5
+    restart: on-failure:5
-  #   environment:
+    environment:
-  #     - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-test123}
-  #     - DATA_SOURCE_URI=postgres?sslmode=disable
+      - DATA_SOURCE_URI=postgres1:5432/postgres?sslmode=disable
-  #     - DATA_SOURCE_USER=${POSTGRES_USER:-postgres}
+      - DATA_SOURCE_USER=${POSTGRES_USER:-postgres}
-  #     - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123}
+      - DATA_SOURCE_PASS=${POSTGRES_PASSWORD:-test123}
-  #     - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml
+      - PG_EXPORTER_EXTEND_QUERY_PATH=/etc/pgexporter/queries.yml
-  #   volumes:
+    volumes:
-  #     - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z
+      - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml:Z
-  #     - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z
+      - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml:Z
-  #   command:
+    command:
-  #     # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info
+      # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info
-  #     - --config.file=/etc/pgexporter/postgres-exporter.yml
+      - --config.file=/etc/pgexporter/postgres-exporter.yml
-  #   depends_on:
+    depends_on:
-  #     - postgres
+      postgres1:
-  #   networks:
+        condition: service_healthy
-  #     - simulation
+    networks:
+      - simulation
 
   prometheus:
     image: prom/prometheus:latest

monitoring/configuration/dashboards/nwaku-store.json

@@ -270,12 +270,12 @@
         "overrides": []
       },
       "gridPos": {
-        "h": 8,
+        "h": 6,
         "w": 12,
-        "x": 0,
+        "x": 12,
-        "y": 8
+        "y": 60
       },
-      "id": 4,
+      "id": 38,
       "options": {
         "legend": {
           "calcs": [],
@@ -295,15 +295,247 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "waku_legacy_store_queries",
+          "exemplar": true,
-          "instant": false,
+          "expr": "waku_store_messages",
-          "legendFormat": "{{instance}}_{{__name__}}",
+          "interval": "",
+          "legendFormat": "{{type}}",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Legacy store queries",
+      "title": "Waku Store Messages",
       "type": "timeseries"
+    },
+    {
+      "cards": {},
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateRdYlGn",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "scaleDistribution": {
+              "type": "linear"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "heatmap": {},
+      "hideZeroBuckets": true,
+      "highlightCards": true,
+      "id": 77,
+      "legend": {
+        "show": false
+      },
+      "maxDataPoints": 120,
+      "options": {
+        "calculate": false,
+        "calculation": {},
+        "cellGap": 2,
+        "cellValues": {},
+        "color": {
+          "exponent": 0.5,
+          "fill": "#b4ff00",
+          "mode": "scheme",
+          "reverse": false,
+          "scale": "exponential",
+          "scheme": "RdYlGn",
+          "steps": 128
+        },
+        "exemplars": {
+          "color": "rgba(255,0,255,0.7)"
+        },
+        "filterValues": {
+          "le": 1e-9
+        },
+        "legend": {
+          "show": false
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "showValue": "never",
+        "tooltip": {
+          "mode": "single",
+          "showColorScale": false,
+          "yHistogram": false
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "decimals": 0,
+          "reverse": false,
+          "unit": "s"
+        }
+      },
+      "pluginVersion": "10.4.2",
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "exemplar": false,
+          "expr": "sum by (le)(rate(waku_store_query_duration_seconds_bucket[$__rate_interval]))",
+          "format": "heatmap",
+          "interval": "",
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Store Query Duration",
+      "tooltip": {
+        "show": true,
+        "showHistogram": false
+      },
+      "type": "heatmap",
+      "xAxis": {
+        "show": true
+      },
+      "yAxis": {
+        "decimals": 0,
+        "format": "s",
+        "logBase": 1,
+        "show": true
+      },
+      "yBucketBound": "auto"
+    },
+    {
+      "cards": {},
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateRdYlGn",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "scaleDistribution": {
+              "type": "linear"
+            }
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "heatmap": {},
+      "hideZeroBuckets": true,
+      "highlightCards": true,
+      "id": 75,
+      "legend": {
+        "show": false
+      },
+      "maxDataPoints": 120,
+      "options": {
+        "calculate": false,
+        "calculation": {},
+        "cellGap": 2,
+        "cellValues": {},
+        "color": {
+          "exponent": 0.5,
+          "fill": "#b4ff00",
+          "mode": "scheme",
+          "reverse": false,
+          "scale": "exponential",
+          "scheme": "RdYlGn",
+          "steps": 128
+        },
+        "exemplars": {
+          "color": "rgba(255,0,255,0.7)"
+        },
+        "filterValues": {
+          "le": 1e-9
+        },
+        "legend": {
+          "show": false
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "showValue": "never",
+        "tooltip": {
+          "mode": "single",
+          "showColorScale": false,
+          "yHistogram": false
+        },
+        "yAxis": {
+          "axisPlacement": "left",
+          "decimals": 0,
+          "reverse": false,
+          "unit": "s"
+        }
+      },
+      "pluginVersion": "10.4.2",
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "exemplar": false,
+          "expr": "sum by (le)(rate(waku_store_insert_duration_seconds_bucket[$__rate_interval]))",
+          "format": "heatmap",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "{{le}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Store Insert Duration",
+      "tooltip": {
+        "show": true,
+        "showHistogram": false
+      },
+      "type": "heatmap",
+      "xAxis": {
+        "show": true
+      },
+      "yAxis": {
+        "decimals": 0,
+        "format": "s",
+        "logBase": 1,
+        "show": true
+      },
+      "yBucketBound": "auto"
     }
   ],
   "refresh": "",

monitoring/prometheus-config.yml

@@ -17,8 +17,8 @@ scrape_configs:
     static_configs:
     - targets:
       - bootstrap:8008
-      - waku-simulator_nwaku_store_pg1_1:8008
+      - waku-simulator_nwaku_store_pg1_1:8009
-      - waku-simulator_nwaku_store_pg2_1:8008
+      - waku-simulator_nwaku_store_pg2_1:8010
       - waku-simulator_nwaku_1:8008
       - waku-simulator_nwaku_2:8008
       - waku-simulator_nwaku_3:8008

run_nwaku_static1.sh

@@ -1,113 +0,0 @@
-#!/bin/sh
-
-# Check Linux Distro Version - it can differ depending on the nwaku image used
-# Install bind-tools/dnsutils package used for domain name resolution
-OS=$(cat /etc/os-release)
-if echo $OS | grep -q "Debian"; then
-    echo "The operating system is Debian."
-    apt update
-    apt install -y dnsutils
-elif echo $OS | grep -q "Alpine"; then
-    echo "The operating system is Alpine."
-    apk add bind-tools
-fi
-
-if test -f .env; then
-  echo "Using .env file"  
-  . $(pwd)/.env
-fi
-
-# Function to extract IP address from URL, resolve the IP and replace it in the original URL
-get_ip_address_and_replace() {
-    local url=$1
-    local domain_name=$(echo $RPC_URL | awk -F[/:] '{print $4}')
-    local ip_address=$(dig +short $domain_name)
-    valid_rpc_url="$(echo "$url" | sed "s/$domain_name/$ip_address/g")"
-    echo $valid_rpc_url
-}
-
-# the format of the RPC URL is checked in the generateRlnKeystore command and hostnames are not valid
-pattern="^(https?):\/\/((localhost)|([\w_-]+(?:(?:\.[\w_-]+)+)))(:[0-9]{1,5})?([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])*"
-# Perform regex matching
-if echo "$RPC_URL" | grep -q "$pattern"; then
-    echo "RPC URL is valid"
-else
-    echo "RPC URL is invalid: $RPC_URL. Attempting to resolve hostname."
-    resolved_rpc_url="$(get_ip_address_and_replace $RPC_URL)"
-    if [ -z "$resolved_rpc_url" ]; then
-        echo -e "Failed to retrieve IP address for $RPC_URL\n"
-    else
-        echo -e "Resolved RPC URL for $RPC_URL: $resolved_rpc_url"
-        RPC_URL="$resolved_rpc_url"
-    fi
-fi
-
-if test -f .$RLN_CREDENTIAL_PATH; then
-  echo "$RLN_CREDENTIAL_PATH already exists. Use it instead of creating a new one."
-else
-  echo "Generating RLN keystore..."
-  /usr/bin/wakunode generateRlnKeystore \
-    --rln-relay-eth-client-address="$RPC_URL" \
-    --rln-relay-eth-private-key=$PRIVATE_KEY  \
-    --rln-relay-eth-contract-address=$RLN_CONTRACT_ADDRESS \
-    --rln-relay-cred-path=$RLN_CREDENTIAL_PATH \
-    --rln-relay-cred-password=$RLN_CREDENTIAL_PASSWORD \
-    --rln-relay-user-message-limit=$RLN_RELAY_MSG_LIMIT \
-    --log-level=DEBUG \
-    --execute
-fi
-
-IP=$(ip a | grep "inet " | grep -Fv 127.0.0.1 | sed 's/.*inet \([^/]*\).*/\1/')
-
-echo "I am a nwaku node"
-
-# Get an unique node index based on the container's IP
-FOURTH_OCTET=${IP##*.}
-THIRD_OCTET="${IP%.*}"; THIRD_OCTET="${THIRD_OCTET##*.}"
-NODE_INDEX=$((FOURTH_OCTET + 256 * THIRD_OCTET))
-
-echo "NODE_INDEX $NODE_INDEX"
-
-RETRIES=${RETRIES:=10}
-
-while [ -z "${BOOTSTRAP_ENR}" ] && [ ${RETRIES} -ge 0 ]; do
-  BOOTSTRAP_ENR=$(wget -qO- http://bootstrap:8645/debug/v1/info --header='Content-Type:application/json' 2> /dev/null | sed 's/.*"enrUri":"\([^"]*\)".*/\1/');
-  echo "Bootstrap node not ready, retrying (retries left: ${RETRIES})"
-  sleep 1
-  RETRIES=$(( $RETRIES - 1 ))
-done
-
-if [ -z "${BOOTSTRAP_ENR}" ]; then
-   echo "Could not get BOOTSTRAP_ENR and none provided. Failing"
-   exit 1
-fi
-
-echo "Using bootstrap node: ${BOOTSTRAP_ENR}"
-exec /usr/bin/wakunode\
-      --relay=true\
-      --max-connections=250\
-      --rest=true\
-      --rest-admin=true\
-      --rest-private=true\
-      --rest-address=0.0.0.0\
-      --cluster-id=0\
-      --pubsub-topic=/waku/2/default-waku/proto\
-      --rest-port=8645\
-      --rln-relay=true\
-      --rln-relay-dynamic=true\
-      --rln-relay-eth-client-address="$RPC_URL"\
-      --rln-relay-eth-contract-address=$RLN_CONTRACT_ADDRESS\
-      --rln-relay-cred-path=$RLN_CREDENTIAL_PATH\
-      --rln-relay-cred-password=$RLN_CREDENTIAL_PASSWORD\
-      --rln-relay-tree-path="rlnv2_tree1"\
-      --rln-relay-epoch-sec=$RLN_RELAY_EPOCH_SEC\
-      --rln-relay-user-message-limit=$RLN_RELAY_MSG_LIMIT\
-      --dns-discovery=true\
-      --discv5-discovery=true\
-      --discv5-enr-auto-update=True\
-      --log-level=DEBUG\
-      --metrics-server=True\
-      --metrics-server-address=0.0.0.0\
-      --discv5-bootstrap-node=${BOOTSTRAP_ENR}\
-      --nat=extip:${IP}\
-      --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942df25

run_nwaku_store_postgres1.sh

@@ -39,6 +39,6 @@ exec /usr/bin/wakunode\
       --store-message-retention-policy=size:20MB\
       --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres1:5432/postgres"\
       --store-sync=true\
-      --store-sync-interval=120\
+      --store-sync-interval=200\
       --store-sync-range=3600\
       --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942daa1

run_nwaku_store_postgres2.sh

@@ -28,7 +28,7 @@ exec /usr/bin/wakunode\
       --dns-discovery=true\
       --discv5-discovery=true\
       --discv5-enr-auto-update=True\
-      --log-level=TRACE\
+      --log-level=DEBUG\
       --metrics-server=True\
       --metrics-server-address=0.0.0.0\
       --discv5-bootstrap-node=${BOOTSTRAP_ENR}\
@@ -37,9 +37,9 @@ exec /usr/bin/wakunode\
       --ports-shift=2\
       --store=true\
       --store-message-retention-policy=size:40MB\
+      --nat=extip:${IP}\
       --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres2:5432/postgres"\
       --store-sync=true\
-      --store-sync-interval=80\
+      --store-sync-interval=60\
       --store-sync-range=3600\
-      --staticnode=/ip4/10.2.0.100/tcp/60001/p2p/16Uiu2HAm3iFcvPDcYge7JFev3ZRdsiUoYFwxtzGZW1BSoV4b4Gc6\
       --nodekey=5978783f8b1a16795032371fff7a526af352d9dca38179af7d71c0122942daa3