diff --git a/codexcrawler/components/chainmetrics.nim b/codexcrawler/components/chainmetrics.nim index b23041c..4956ba0 100644 --- a/codexcrawler/components/chainmetrics.nim +++ b/codexcrawler/components/chainmetrics.nim @@ -67,7 +67,7 @@ method start*(c: ChainMetrics): Future[?!void] {.async.} = return await c.step() if c.state.config.marketplaceEnable: - await c.state.whileRunning(onStep, 10.minutes) + await c.state.whileRunning(onStep, c.state.config.requestCheckDelay.minutes) return success() diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim index 10f434b..47fca12 100644 --- a/codexcrawler/config.nim +++ b/codexcrawler/config.nim @@ -10,7 +10,7 @@ let doc = Codex Network Crawler. Generates network metrics. Usage: - codexcrawler [--logLevel=] [--publicIp=] [--metricsAddress=] [--metricsPort=

] [--dataDir=

] [--discoveryPort=

] [--bootNodes=] [--dhtEnable=] [--stepDelay=] [--revisitDelay=] [--checkDelay=] [--expiryDelay=] [--marketplaceEnable=] [--ethProvider=] [--marketplaceAddress=] + codexcrawler [--logLevel=] [--publicIp=] [--metricsAddress=] [--metricsPort=

] [--dataDir=

] [--discoveryPort=

] [--bootNodes=] [--dhtEnable=] [--stepDelay=] [--revisitDelay=] [--checkDelay=] [--expiryDelay=] [--marketplaceEnable=] [--ethProvider=] [--marketplaceAddress=] [--requestCheckDelay=] Options: --logLevel= Sets log level [default: INFO] @@ -20,14 +20,17 @@ Options: --dataDir=

Directory for storing data [default: crawler_data] --discoveryPort=

Port used for DHT [default: 8090] --bootNodes= Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs] + --dhtEnable= Set to "1" to enable DHT crawler [default: 0] --stepDelay= Delay in milliseconds per node visit [default: 1000] --revisitDelay= Delay in minutes after which a node can be revisited [default: 60] --checkDelay= Delay with which the 'revisitDelay' is checked for all known nodes [default: 10] --expiryDelay= Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h) + --marketplaceEnable= Set to "1" to enable marketplace metrics [default: 1] --ethProvider= Address including http(s) or ws of the eth provider --marketplaceAddress= Eth address of Codex contracts deployment + --requestCheckDelay= Delay in minutes after which storage contract status is (re)checked [default: 10] """ import strutils @@ -51,6 +54,7 @@ type Config* = ref object marketplaceEnable*: bool ethProvider*: string marketplaceAddress*: string + requestCheckDelay*: int proc `$`*(config: Config): string = "Crawler:" & " logLevel=" & config.logLevel & " publicIp=" & config.publicIp & @@ -61,7 +65,7 @@ proc `$`*(config: Config): string = " expiryDelayMins=" & $config.expiryDelayMins & " checkDelayMins=" & $config.checkDelayMins & " marketplaceEnable=" & $config.marketplaceEnable & " ethProvider=" & config.ethProvider & " marketplaceAddress=" & - config.marketplaceAddress + config.marketplaceAddress & " requestCheckDelay=" & $config.requestCheckDelay proc getDefaultTestnetBootNodes(): seq[string] = @[ @@ -124,4 +128,5 @@ proc parseConfig*(): Config = marketplaceEnable: getEnable(get("--marketplaceEnable")), ethProvider: get("--ethProvider"), marketplaceAddress: get("--marketplaceAddress"), + requestCheckDelay: parseInt(get("--requestCheckDelay")) ) diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index a9846a4..d7b1abc 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -19,9 +19,10 @@ EXPIRYDELAY=${CRAWLER_EXPIRYDELAY:-1440} MARKETPLACEENABLE=${CRAWLER_MARKETPLACEENABLE:-1} ETHPROVIDER=${CRAWLER_ETHPROVIDER:-NULL} MARKETPLACEADDRESS=${CRAWLER_MARKETPLACEADDRESS:-NULL} +REQUESTCHECKDELAY=${CRAWLER_REQUESTCHECKDELAY:-10} # Update CLI arguments -set -- "$@" --logLevel="${LOGLEVEL}" --publicIp="${PUBLICIP}" --metricsAddress="${METRICSADDRESS}" --metricsPort="${METRICSPORT}" --dataDir="${DATADIR}" --discoveryPort="${DISCPORT}" --bootNodes="${BOOTNODES}" --dhtEnable="${DHTENABLE}" --stepDelay="${STEPDELAY}" --revisitDelay="${REVISITDELAY}" --expiryDelay="${EXPIRYDELAY}" --checkDelay="${CHECKDELAY}" --marketplaceEnable="${MARKETPLACEENABLE}" --ethProvider="${ETHPROVIDER}" --marketplaceAddress="${MARKETPLACEADDRESS}" +set -- "$@" --logLevel="${LOGLEVEL}" --publicIp="${PUBLICIP}" --metricsAddress="${METRICSADDRESS}" --metricsPort="${METRICSPORT}" --dataDir="${DATADIR}" --discoveryPort="${DISCPORT}" --bootNodes="${BOOTNODES}" --dhtEnable="${DHTENABLE}" --stepDelay="${STEPDELAY}" --revisitDelay="${REVISITDELAY}" --expiryDelay="${EXPIRYDELAY}" --checkDelay="${CHECKDELAY}" --marketplaceEnable="${MARKETPLACEENABLE}" --ethProvider="${ETHPROVIDER}" --marketplaceAddress="${MARKETPLACEADDRESS}" --requestCheckDelay="${REQUESTCHECKDELAY}" # Run echo "Run Codex Crawler" diff --git a/tests/codexcrawler/components/testchainmetrics.nim b/tests/codexcrawler/components/testchainmetrics.nim index 1098f45..4d279dd 100644 --- a/tests/codexcrawler/components/testchainmetrics.nim +++ b/tests/codexcrawler/components/testchainmetrics.nim @@ -39,10 +39,10 @@ suite "ChainMetrics": proc onStep() {.async.} = (await state.steppers[0]()).tryGet() - test "start should start stepper for 10 minutes": + test "start should start stepper for config.requestCheckDelay minutes": check: state.delays.len == 1 - state.delays[0] == 10.minutes + state.delays[0] == state.config.requestCheckDelay.minutes test "onStep should remove old non-running requests from request store": let rid = genRid() diff --git a/tests/codexcrawler/mocks/mockstate.nim b/tests/codexcrawler/mocks/mockstate.nim index e61ae92..cefe98d 100644 --- a/tests/codexcrawler/mocks/mockstate.nim +++ b/tests/codexcrawler/mocks/mockstate.nim @@ -22,7 +22,11 @@ method whileRunning*(s: MockState, step: OnStep, delay: Duration) {.async.} = proc createMockState*(): MockState = MockState( status: ApplicationStatus.Running, - config: Config(dhtEnable: true, marketplaceEnable: true), + config: Config( + dhtEnable: true, + marketplaceEnable: true, + requestCheckDelay: 4 + ), events: Events( nodesFound: newAsyncDataEvent[seq[Nid]](), newNodesDiscovered: newAsyncDataEvent[seq[Nid]](),