mirror of
https://github.com/logos-storage/logos-storage-network-crawler.git
synced 2026-01-02 13:33:08 +00:00
setting up container script
This commit is contained in:
Ben
parent
37550589b9
commit
f1dfd7b052
@ -13,8 +13,8 @@ Usage:
|
||||
codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--stepDelay=<ms>] [--revisitDelay=<m>]
|
||||
|
||||
Options:
|
||||
--logLevel=<l> Sets log level [default: TRACE]
|
||||
--publicIp=<a> Public IP address where this instance is reachable.
|
||||
--logLevel=<l> Sets log level [default: INFO]
|
||||
--metricsAddress=<ip> Listen address of the metrics server [default: 0.0.0.0]
|
||||
--metricsPort=<p> Listen HTTP port of the metrics server [default: 8008]
|
||||
--dataDir=<dir> Directory for storing data [default: crawler_data]
|
||||
|
||||
@ -1,44 +1,18 @@
|
||||
services:
|
||||
codex-node1:
|
||||
image: codexstorage/nim-codex:sha-82b0399
|
||||
crawler1:
|
||||
build: crawler.Dockerfile
|
||||
environment:
|
||||
- CODEX_LOG_LEVEL=${CODEX_LOG_LEVEL:-TRACE}
|
||||
- CODEX_METRICS=${CODEX_METRICS:-false}
|
||||
- CODEX_METRICS_ADDRESS=${CODEX_METRICS_ADDRESS:-0.0.0.0}
|
||||
- CODEX_METRICS_PORT=${CODEX_METRICS_PORT:-8008}
|
||||
- CODEX_DATA_DIR=${CODEX_DATA_DIR:-/datadir}
|
||||
- CODEX_LISTEN_ADDRS=${CODEX_LISTEN_ADDRS:-/ip4/0.0.0.0/tcp/2345}
|
||||
- CODEX_NAT=${CODEX_NAT:-10.0.0.10}
|
||||
- CODEX_DISC_PORT=${CODEX_DISC_PORT:-8090}
|
||||
- CODEX_NET_PRIVKEY=${CODEX_NET_PRIVKEY:-key}
|
||||
# - CODEX_BOOTSTRAP_NODE=${CODEX_BOOTSTRAP_NODE}
|
||||
- CODEX_MAX_PEERS=${CODEX_MAX_PEERS:-160}
|
||||
- CODEX_AGENT_STRING=${CODEX_AGENT_STRING:-Codex}
|
||||
- CODEX_API_BINDADDR=${CODEX_API_BINDADDR:-0.0.0.0}
|
||||
- CODEX_API_PORT=${CODEX_API_PORT:-8080}
|
||||
- CODEX_REPO_KIND=${CODEX_REPO_KIND:-fs}
|
||||
- CODEX_STORAGE_QUOTA=${CODEX_STORAGE_QUOTA:-8589934592}
|
||||
- CODEX_BLOCK_TTL=${CODEX_BLOCK_TTL:-0}
|
||||
# - CODEX_BLOCK_MI=${CODEX_BLOCK_MI}
|
||||
- CODEX_BLOCK_MN=${CODEX_BLOCK_MN:-1000}
|
||||
- CODEX_CACHE_SIZE=${CODEX_CACHE_SIZE:-0}
|
||||
- CODEX_PERSISTENCE=${CODEX_PERSISTENCE:-false}
|
||||
- CODEX_ETH_PROVIDER=${CODEX_ETH_PROVIDER:-ws://localhost:8545}
|
||||
# - CODEX_ETH_ACCOUNT=${CODEX_ETH_ACCOUNT}
|
||||
# - CODEX_MARKETPLACE_ADDRESS=${CODEX_MARKETPLACE_ADDRESS:-0x59b670e9fA9D0A427751Af201D676719a970857b}
|
||||
- CODEX_VALIDATOR=${CODEX_VALIDATOR:-false}
|
||||
- CODEX_VALIDATOR_MAX_SLOTS=${CODEX_VALIDATOR_MAX_SLOTS:-1000}
|
||||
- NAT_IP_AUTO=false
|
||||
- NAT_PUBLIC_IP_AUTO=https://ipinfo.io/ip
|
||||
- CRAWLER_LOGLEVEL=TRACE
|
||||
# - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage
|
||||
- CRAWLER_METRICSADDRESS=0.0.0.0
|
||||
- CRAWLER_METRICSPORT=8008
|
||||
- CRAWLER_DATADIR=crawler_data
|
||||
- CRAWLER_DISCPORT=8090
|
||||
- CRAWLER_BOOTNODES=testnet_sprs
|
||||
- CRAWLER_STEPDELAY=3000
|
||||
- CRAWLER_REVISITDELAY=1440
|
||||
ports:
|
||||
- 8080:8080/tcp # REST API
|
||||
- 8008:8008/tcp # Metrics
|
||||
- 2345:2345/tcp # libp2p
|
||||
- 8090:8090/udp # DHT discovery
|
||||
volumes:
|
||||
- ./datadir:/datadir:z
|
||||
networks:
|
||||
- codex
|
||||
networks:
|
||||
codex:
|
||||
driver: bridge
|
||||
- ./crawler_data:/crawler_data:z
|
||||
|
||||
@ -1,76 +1,17 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Environment variables from files
|
||||
# If set to file path, read the file and export the variables
|
||||
# If set to directory path, read all files in the directory and export the variables
|
||||
if [[ -n "${ENV_PATH}" ]]; then
|
||||
set -a
|
||||
[[ -f "${ENV_PATH}" ]] && source "${ENV_PATH}" || for f in "${ENV_PATH}"/*; do source "$f"; done
|
||||
set +a
|
||||
fi
|
||||
AUTOPUBLICIP=`curl http://ip.codex.storage`
|
||||
|
||||
# Should be passed if env variable is set.
|
||||
# public IP should fetch ip.codex.storage
|
||||
|
||||
# --logLevel=<l> Sets log level [default: TRACE]
|
||||
# --publicIp=<a> Public IP address where this instance is reachable. [default: 45.82.185.194]
|
||||
# --metricsAddress=<ip> Listen address of the metrics server [default: 0.0.0.0]
|
||||
# --metricsPort=<p> Listen HTTP port of the metrics server [default: 8008]
|
||||
# --dataDir=<dir> Directory for storing data [default: crawler_data]
|
||||
# --discoveryPort=<p> Port used for DHT [default: 8090]
|
||||
# --bootNodes=<n> Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
|
||||
# --stepDelay=<ms> Delay in milliseconds per crawl step [default: 3000]
|
||||
# --revisitDelay=<m> Delay in minutes after which a node can be revisited [default: 1] (24h)
|
||||
|
||||
|
||||
# Parameters
|
||||
if [[ -z "${CODEX_NAT}" ]]; then
|
||||
if [[ "${NAT_IP_AUTO}" == "true" && -z "${NAT_PUBLIC_IP_AUTO}" ]]; then
|
||||
export CODEX_NAT="extip:$(hostname --ip-address)"
|
||||
echo "Private: CODEX_NAT=${CODEX_NAT}"
|
||||
elif [[ -n "${NAT_PUBLIC_IP_AUTO}" ]]; then
|
||||
# Run for 60 seconds if fail
|
||||
WAIT=120
|
||||
SECONDS=0
|
||||
SLEEP=5
|
||||
while (( SECONDS < WAIT )); do
|
||||
IP=$(curl -s -f -m 5 "${NAT_PUBLIC_IP_AUTO}")
|
||||
# Check if exit code is 0 and returned value is not empty
|
||||
if [[ $? -eq 0 && -n "${IP}" ]]; then
|
||||
export CODEX_NAT="extip:${IP}"
|
||||
echo "Public: CODEX_NAT=${CODEX_NAT}"
|
||||
break
|
||||
else
|
||||
# Sleep and check again
|
||||
echo "Can't get Public IP - Retry in $SLEEP seconds / $((WAIT - SECONDS))"
|
||||
sleep $SLEEP
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# Stop Codex run if can't get NAT IP when requested
|
||||
if [[ "${NAT_IP_AUTO}" == "true" && -z "${CODEX_NAT}" ]]; then
|
||||
echo "Can't get Private IP - Stop Codex run"
|
||||
exit 1
|
||||
elif [[ -n "${NAT_PUBLIC_IP_AUTO}" && -z "${CODEX_NAT}" ]]; then
|
||||
echo "Can't get Public IP in $WAIT seconds - Stop Codex run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If marketplace is enabled from the testing environment,
|
||||
# The file has to be written before Codex starts.
|
||||
for key in PRIV_KEY ETH_PRIVATE_KEY; do
|
||||
keyfile="private.key"
|
||||
if [[ -n "${!key}" ]]; then
|
||||
[[ "${key}" == "PRIV_KEY" ]] && echo "PRIV_KEY variable is deprecated and will be removed in the next releases, please use ETH_PRIVATE_KEY instead!"
|
||||
echo "${!key}" > "${keyfile}"
|
||||
chmod 600 "${keyfile}"
|
||||
export CODEX_ETH_PRIVATE_KEY="${keyfile}"
|
||||
echo "Private key set"
|
||||
fi
|
||||
done
|
||||
LOGLEVEL=${CRAWLER_LOGLEVEL:-INFO}
|
||||
PUBLICIP=${CRAWLER_PUBLICIP:-${AUTOPUBLICIP}}
|
||||
METRICSADDRESS=${CRAWLER_METRICSADDRESS:-0.0.0.0}
|
||||
METRICSPORT=${CRAWLER_METRICSPORT:-8008}
|
||||
DATADIR=${CRAWLER_DATADIR:-crawler_data}
|
||||
DISCPORT=${CRAWLER_DISCPORT:-8090}
|
||||
BOOTNODES=${CRAWLER_BOOTNODES:-testnet_sprs}
|
||||
STEPDELAY=${CRAWLER_STEPDELAY:-1000}
|
||||
REVISITDELAY=${CRAWLER_REVISITDELAY:-1440}
|
||||
|
||||
# Run
|
||||
echo "Run Codex node"
|
||||
exec "$@"
|
||||
exec "--logLevel=${LOGLEVEL} --publicIp=${PUBLICIP} --metricsAddress=${METRICSADDRESS} --metricsPort=${METRICSPORT} --dataDir=${DATADIR} --discoveryPort=${DISCPORT} --bootNodes=${BOOTNODES} --stepDelay=${STEPDELAY} --revisitDelay=${REVISITDELAY}"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user