diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim index 1c1f041..60541c2 100644 --- a/codexcrawler/config.nim +++ b/codexcrawler/config.nim @@ -13,8 +13,8 @@ Usage: codexcrawler [--logLevel=] [--publicIp=] [--metricsAddress=] [--metricsPort=

] [--dataDir=

] [--discoveryPort=

] [--bootNodes=] [--stepDelay=] [--revisitDelay=] Options: - --logLevel= Sets log level [default: TRACE] --publicIp= Public IP address where this instance is reachable. + --logLevel= Sets log level [default: INFO] --metricsAddress= Listen address of the metrics server [default: 0.0.0.0] --metricsPort=

Listen HTTP port of the metrics server [default: 8008] --dataDir=

Directory for storing data [default: crawler_data] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 9ffdd82..36ccd80 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,44 +1,18 @@ services: - codex-node1: - image: codexstorage/nim-codex:sha-82b0399 + crawler1: + build: crawler.Dockerfile environment: - - CODEX_LOG_LEVEL=${CODEX_LOG_LEVEL:-TRACE} - - CODEX_METRICS=${CODEX_METRICS:-false} - - CODEX_METRICS_ADDRESS=${CODEX_METRICS_ADDRESS:-0.0.0.0} - - CODEX_METRICS_PORT=${CODEX_METRICS_PORT:-8008} - - CODEX_DATA_DIR=${CODEX_DATA_DIR:-/datadir} - - CODEX_LISTEN_ADDRS=${CODEX_LISTEN_ADDRS:-/ip4/0.0.0.0/tcp/2345} - - CODEX_NAT=${CODEX_NAT:-10.0.0.10} - - CODEX_DISC_PORT=${CODEX_DISC_PORT:-8090} - - CODEX_NET_PRIVKEY=${CODEX_NET_PRIVKEY:-key} - # - CODEX_BOOTSTRAP_NODE=${CODEX_BOOTSTRAP_NODE} - - CODEX_MAX_PEERS=${CODEX_MAX_PEERS:-160} - - CODEX_AGENT_STRING=${CODEX_AGENT_STRING:-Codex} - - CODEX_API_BINDADDR=${CODEX_API_BINDADDR:-0.0.0.0} - - CODEX_API_PORT=${CODEX_API_PORT:-8080} - - CODEX_REPO_KIND=${CODEX_REPO_KIND:-fs} - - CODEX_STORAGE_QUOTA=${CODEX_STORAGE_QUOTA:-8589934592} - - CODEX_BLOCK_TTL=${CODEX_BLOCK_TTL:-0} - # - CODEX_BLOCK_MI=${CODEX_BLOCK_MI} - - CODEX_BLOCK_MN=${CODEX_BLOCK_MN:-1000} - - CODEX_CACHE_SIZE=${CODEX_CACHE_SIZE:-0} - - CODEX_PERSISTENCE=${CODEX_PERSISTENCE:-false} - - CODEX_ETH_PROVIDER=${CODEX_ETH_PROVIDER:-ws://localhost:8545} - # - CODEX_ETH_ACCOUNT=${CODEX_ETH_ACCOUNT} - # - CODEX_MARKETPLACE_ADDRESS=${CODEX_MARKETPLACE_ADDRESS:-0x59b670e9fA9D0A427751Af201D676719a970857b} - - CODEX_VALIDATOR=${CODEX_VALIDATOR:-false} - - CODEX_VALIDATOR_MAX_SLOTS=${CODEX_VALIDATOR_MAX_SLOTS:-1000} - - NAT_IP_AUTO=false - - NAT_PUBLIC_IP_AUTO=https://ipinfo.io/ip + - CRAWLER_LOGLEVEL=TRACE + # - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage + - CRAWLER_METRICSADDRESS=0.0.0.0 + - CRAWLER_METRICSPORT=8008 + - CRAWLER_DATADIR=crawler_data + - CRAWLER_DISCPORT=8090 + - CRAWLER_BOOTNODES=testnet_sprs + - CRAWLER_STEPDELAY=3000 + - CRAWLER_REVISITDELAY=1440 ports: - - 8080:8080/tcp # REST API - 8008:8008/tcp # Metrics - - 2345:2345/tcp # libp2p - 8090:8090/udp # DHT discovery volumes: - - ./datadir:/datadir:z - networks: - - codex -networks: - codex: - driver: bridge + - ./crawler_data:/crawler_data:z diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 5618499..e49057e 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,76 +1,17 @@ #!/bin/bash -# Environment variables from files -# If set to file path, read the file and export the variables -# If set to directory path, read all files in the directory and export the variables -if [[ -n "${ENV_PATH}" ]]; then - set -a - [[ -f "${ENV_PATH}" ]] && source "${ENV_PATH}" || for f in "${ENV_PATH}"/*; do source "$f"; done - set +a -fi +AUTOPUBLICIP=`curl http://ip.codex.storage` -# Should be passed if env variable is set. -# public IP should fetch ip.codex.storage - -# --logLevel= Sets log level [default: TRACE] -# --publicIp= Public IP address where this instance is reachable. [default: 45.82.185.194] -# --metricsAddress= Listen address of the metrics server [default: 0.0.0.0] -# --metricsPort=

Listen HTTP port of the metrics server [default: 8008] -# --dataDir=

Directory for storing data [default: crawler_data] -# --discoveryPort=

Port used for DHT [default: 8090] -# --bootNodes= Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs] -# --stepDelay= Delay in milliseconds per crawl step [default: 3000] -# --revisitDelay= Delay in minutes after which a node can be revisited [default: 1] (24h) - - -# Parameters -if [[ -z "${CODEX_NAT}" ]]; then - if [[ "${NAT_IP_AUTO}" == "true" && -z "${NAT_PUBLIC_IP_AUTO}" ]]; then - export CODEX_NAT="extip:$(hostname --ip-address)" - echo "Private: CODEX_NAT=${CODEX_NAT}" - elif [[ -n "${NAT_PUBLIC_IP_AUTO}" ]]; then - # Run for 60 seconds if fail - WAIT=120 - SECONDS=0 - SLEEP=5 - while (( SECONDS < WAIT )); do - IP=$(curl -s -f -m 5 "${NAT_PUBLIC_IP_AUTO}") - # Check if exit code is 0 and returned value is not empty - if [[ $? -eq 0 && -n "${IP}" ]]; then - export CODEX_NAT="extip:${IP}" - echo "Public: CODEX_NAT=${CODEX_NAT}" - break - else - # Sleep and check again - echo "Can't get Public IP - Retry in $SLEEP seconds / $((WAIT - SECONDS))" - sleep $SLEEP - fi - done - fi -fi - -# Stop Codex run if can't get NAT IP when requested -if [[ "${NAT_IP_AUTO}" == "true" && -z "${CODEX_NAT}" ]]; then - echo "Can't get Private IP - Stop Codex run" - exit 1 -elif [[ -n "${NAT_PUBLIC_IP_AUTO}" && -z "${CODEX_NAT}" ]]; then - echo "Can't get Public IP in $WAIT seconds - Stop Codex run" - exit 1 -fi - -# If marketplace is enabled from the testing environment, -# The file has to be written before Codex starts. -for key in PRIV_KEY ETH_PRIVATE_KEY; do - keyfile="private.key" - if [[ -n "${!key}" ]]; then - [[ "${key}" == "PRIV_KEY" ]] && echo "PRIV_KEY variable is deprecated and will be removed in the next releases, please use ETH_PRIVATE_KEY instead!" - echo "${!key}" > "${keyfile}" - chmod 600 "${keyfile}" - export CODEX_ETH_PRIVATE_KEY="${keyfile}" - echo "Private key set" - fi -done +LOGLEVEL=${CRAWLER_LOGLEVEL:-INFO} +PUBLICIP=${CRAWLER_PUBLICIP:-${AUTOPUBLICIP}} +METRICSADDRESS=${CRAWLER_METRICSADDRESS:-0.0.0.0} +METRICSPORT=${CRAWLER_METRICSPORT:-8008} +DATADIR=${CRAWLER_DATADIR:-crawler_data} +DISCPORT=${CRAWLER_DISCPORT:-8090} +BOOTNODES=${CRAWLER_BOOTNODES:-testnet_sprs} +STEPDELAY=${CRAWLER_STEPDELAY:-1000} +REVISITDELAY=${CRAWLER_REVISITDELAY:-1440} # Run echo "Run Codex node" -exec "$@" +exec "--logLevel=${LOGLEVEL} --publicIp=${PUBLICIP} --metricsAddress=${METRICSADDRESS} --metricsPort=${METRICSPORT} --dataDir=${DATADIR} --discoveryPort=${DISCPORT} --bootNodes=${BOOTNODES} --stepDelay=${STEPDELAY} --revisitDelay=${REVISITDELAY}"