setting up container script

codexcrawler/config.nim

@@ -13,8 +13,8 @@ Usage:
   codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--stepDelay=<ms>] [--revisitDelay=<m>]
 
 Options:
-  --logLevel=<l>          Sets log level [default: TRACE]
   --publicIp=<a>          Public IP address where this instance is reachable.
+  --logLevel=<l>          Sets log level [default: INFO]
   --metricsAddress=<ip>   Listen address of the metrics server [default: 0.0.0.0]
   --metricsPort=<p>       Listen HTTP port of the metrics server [default: 8008]
   --dataDir=<dir>         Directory for storing data [default: crawler_data]

docker/docker-compose.yaml

@@ -1,44 +1,18 @@
 services:
-  codex-node1:
+  crawler1:
-    image: codexstorage/nim-codex:sha-82b0399
+    build: crawler.Dockerfile
     environment:
-      - CODEX_LOG_LEVEL=${CODEX_LOG_LEVEL:-TRACE}
+      - CRAWLER_LOGLEVEL=TRACE
-      - CODEX_METRICS=${CODEX_METRICS:-false}
+      # - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage
-      - CODEX_METRICS_ADDRESS=${CODEX_METRICS_ADDRESS:-0.0.0.0}
+      - CRAWLER_METRICSADDRESS=0.0.0.0
-      - CODEX_METRICS_PORT=${CODEX_METRICS_PORT:-8008}
+      - CRAWLER_METRICSPORT=8008
-      - CODEX_DATA_DIR=${CODEX_DATA_DIR:-/datadir}
+      - CRAWLER_DATADIR=crawler_data
-      - CODEX_LISTEN_ADDRS=${CODEX_LISTEN_ADDRS:-/ip4/0.0.0.0/tcp/2345}
+      - CRAWLER_DISCPORT=8090
-      - CODEX_NAT=${CODEX_NAT:-10.0.0.10}
+      - CRAWLER_BOOTNODES=testnet_sprs
-      - CODEX_DISC_PORT=${CODEX_DISC_PORT:-8090}
+      - CRAWLER_STEPDELAY=3000
-      - CODEX_NET_PRIVKEY=${CODEX_NET_PRIVKEY:-key}
+      - CRAWLER_REVISITDELAY=1440
-      # - CODEX_BOOTSTRAP_NODE=${CODEX_BOOTSTRAP_NODE}
-      - CODEX_MAX_PEERS=${CODEX_MAX_PEERS:-160}
-      - CODEX_AGENT_STRING=${CODEX_AGENT_STRING:-Codex}
-      - CODEX_API_BINDADDR=${CODEX_API_BINDADDR:-0.0.0.0}
-      - CODEX_API_PORT=${CODEX_API_PORT:-8080}
-      - CODEX_REPO_KIND=${CODEX_REPO_KIND:-fs}
-      - CODEX_STORAGE_QUOTA=${CODEX_STORAGE_QUOTA:-8589934592}
-      - CODEX_BLOCK_TTL=${CODEX_BLOCK_TTL:-0}
-      # - CODEX_BLOCK_MI=${CODEX_BLOCK_MI}
-      - CODEX_BLOCK_MN=${CODEX_BLOCK_MN:-1000}
-      - CODEX_CACHE_SIZE=${CODEX_CACHE_SIZE:-0}
-      - CODEX_PERSISTENCE=${CODEX_PERSISTENCE:-false}
-      - CODEX_ETH_PROVIDER=${CODEX_ETH_PROVIDER:-ws://localhost:8545}
-      # - CODEX_ETH_ACCOUNT=${CODEX_ETH_ACCOUNT}
-      # - CODEX_MARKETPLACE_ADDRESS=${CODEX_MARKETPLACE_ADDRESS:-0x59b670e9fA9D0A427751Af201D676719a970857b}
-      - CODEX_VALIDATOR=${CODEX_VALIDATOR:-false}
-      - CODEX_VALIDATOR_MAX_SLOTS=${CODEX_VALIDATOR_MAX_SLOTS:-1000}
-      - NAT_IP_AUTO=false
-      - NAT_PUBLIC_IP_AUTO=https://ipinfo.io/ip
     ports:
-      - 8080:8080/tcp # REST API
       - 8008:8008/tcp # Metrics
-      - 2345:2345/tcp # libp2p
       - 8090:8090/udp # DHT discovery
     volumes:
-      - ./datadir:/datadir:z
+      - ./crawler_data:/crawler_data:z
-    networks:
-      - codex
-networks:
-  codex:
-    driver: bridge

docker/docker-entrypoint.sh

@@ -1,76 +1,17 @@
 #!/bin/bash
 
-# Environment variables from files
+AUTOPUBLICIP=`curl http://ip.codex.storage`
-# If set to file path, read the file and export the variables
-# If set to directory path, read all files in the directory and export the variables
-if [[ -n "${ENV_PATH}" ]]; then
-  set -a
-  [[ -f "${ENV_PATH}" ]] && source "${ENV_PATH}" || for f in "${ENV_PATH}"/*; do source "$f"; done
-  set +a
-fi
 
-# Should be passed if env variable is set.
+LOGLEVEL=${CRAWLER_LOGLEVEL:-INFO}
-# public IP should fetch ip.codex.storage
+PUBLICIP=${CRAWLER_PUBLICIP:-${AUTOPUBLICIP}}
-
+METRICSADDRESS=${CRAWLER_METRICSADDRESS:-0.0.0.0}
-# --logLevel=<l>          Sets log level [default: TRACE]
+METRICSPORT=${CRAWLER_METRICSPORT:-8008}
-# --publicIp=<a>          Public IP address where this instance is reachable. [default: 45.82.185.194]
+DATADIR=${CRAWLER_DATADIR:-crawler_data}
-# --metricsAddress=<ip>   Listen address of the metrics server [default: 0.0.0.0]
+DISCPORT=${CRAWLER_DISCPORT:-8090}
-# --metricsPort=<p>       Listen HTTP port of the metrics server [default: 8008]
+BOOTNODES=${CRAWLER_BOOTNODES:-testnet_sprs}
-# --dataDir=<dir>         Directory for storing data [default: crawler_data]
+STEPDELAY=${CRAWLER_STEPDELAY:-1000}
-# --discoveryPort=<p>     Port used for DHT [default: 8090]
+REVISITDELAY=${CRAWLER_REVISITDELAY:-1440}
-# --bootNodes=<n>         Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
-# --stepDelay=<ms>        Delay in milliseconds per crawl step [default: 3000]
-# --revisitDelay=<m>      Delay in minutes after which a node can be revisited [default: 1] (24h)
-
-
-# Parameters
-if [[ -z "${CODEX_NAT}" ]]; then
-  if [[ "${NAT_IP_AUTO}" == "true" && -z "${NAT_PUBLIC_IP_AUTO}" ]]; then
-    export CODEX_NAT="extip:$(hostname --ip-address)"
-    echo "Private: CODEX_NAT=${CODEX_NAT}"
-  elif [[ -n "${NAT_PUBLIC_IP_AUTO}" ]]; then
-    # Run for 60 seconds if fail
-    WAIT=120
-    SECONDS=0
-    SLEEP=5
-    while (( SECONDS < WAIT )); do
-      IP=$(curl -s -f -m 5 "${NAT_PUBLIC_IP_AUTO}")
-      # Check if exit code is 0 and returned value is not empty
-      if [[ $? -eq 0 && -n "${IP}" ]]; then
-        export CODEX_NAT="extip:${IP}"
-        echo "Public: CODEX_NAT=${CODEX_NAT}"
-        break
-      else
-        # Sleep and check again
-        echo "Can't get Public IP - Retry in $SLEEP seconds / $((WAIT - SECONDS))"
-        sleep $SLEEP
-      fi
-    done
-  fi
-fi
-
-# Stop Codex run if can't get NAT IP when requested
-if [[ "${NAT_IP_AUTO}" == "true" && -z "${CODEX_NAT}" ]]; then
-  echo "Can't get Private IP - Stop Codex run"
-  exit 1
-elif [[ -n "${NAT_PUBLIC_IP_AUTO}" && -z "${CODEX_NAT}" ]]; then
-  echo "Can't get Public IP in $WAIT seconds - Stop Codex run"
-  exit 1
-fi
-
-# If marketplace is enabled from the testing environment,
-# The file has to be written before Codex starts.
-for key in PRIV_KEY ETH_PRIVATE_KEY; do
-  keyfile="private.key"
-  if [[ -n "${!key}" ]]; then
-    [[ "${key}" == "PRIV_KEY" ]] && echo "PRIV_KEY variable is deprecated and will be removed in the next releases, please use ETH_PRIVATE_KEY instead!"
-    echo "${!key}" > "${keyfile}"
-    chmod 600 "${keyfile}"
-    export CODEX_ETH_PRIVATE_KEY="${keyfile}"
-    echo "Private key set"
-  fi
-done
 
 # Run
 echo "Run Codex node"
-exec "$@"
+exec "--logLevel=${LOGLEVEL} --publicIp=${PUBLICIP} --metricsAddress=${METRICSADDRESS} --metricsPort=${METRICSPORT} --dataDir=${DATADIR} --discoveryPort=${DISCPORT} --bootNodes=${BOOTNODES} --stepDelay=${STEPDELAY} --revisitDelay=${REVISITDELAY}"