diff --git a/.dockerignore b/.dockerignore index 6427da1..49fd67f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ -.github -build -docs -metrics -nimcache -tests +.github +build +docs +metrics +nimcache +tests diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..404d135 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +env.sh text eol=lf +docker/docker-entrypoint.sh text eol=lf diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f9553d5..07eb2f9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,21 +1,21 @@ -name: Docker - - -on: - push: - branches: - - master - tags: - - 'v*.*.*' - workflow_dispatch: - - -jobs: - build-and-push: - name: Build and Push - uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master - with: - docker_file: docker/crawler.Dockerfile - dockerhub_repo: codexstorage/codex-network-crawler - tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} - secrets: inherit +name: Docker + + +on: + push: + branches: + - master + tags: + - 'v*.*.*' + workflow_dispatch: + + +jobs: + build-and-push: + name: Build and Push + uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master + with: + docker_file: docker/crawler.Dockerfile + dockerhub_repo: codexstorage/codex-network-crawler + tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} + secrets: inherit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3a0a7fc..9dfd963 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,43 +1,43 @@ -name: CI - -on: [push, pull_request] - -jobs: - linting: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Check `nph` formatting - uses: arnetheduck/nph-action@v1 - with: - version: 0.6.1 - options: "./" - fail: true - suggest: true - - test: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] - nim: [2.2.4] - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - - name: Build Nim and dependencies - run: | - which gcc - gcc --version - make update - ./env.sh nim --version - make - - - name: Run tests - run: | - make test - - - name: Build crawler - run: | - make +name: CI + +on: [push, pull_request] + +jobs: + linting: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check `nph` formatting + uses: arnetheduck/nph-action@v1 + with: + version: 0.6.1 + options: "./" + fail: true + suggest: true + + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + nim: [2.2.4] + steps: + - uses: actions/checkout@v2 + with: + submodules: true + + - name: Build Nim and dependencies + run: | + which gcc + gcc --version + make update + ./env.sh nim --version + make + + - name: Run tests + run: | + make test + + - name: Build crawler + run: | + make diff --git a/.gitignore b/.gitignore index fbe4999..332b98e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ -* -!*.* -!*/ -nimble.paths -nimbus-build-system.paths -vendor/* -*.exe -crawler_data +* +!*.* +!*/ +nimble.paths +nimbus-build-system.paths +vendor/* +*.exe +crawler_data .update.timestamp diff --git a/Makefile b/Makefile index 4268fad..8b3b755 100644 --- a/Makefile +++ b/Makefile @@ -1,197 +1,197 @@ -# Copyright (c) 2020 Status Research & Development GmbH. Licensed under -# either of: -# - Apache License, version 2.0 -# - MIT license -# at your option. This file may not be copied, modified, or distributed except -# according to those terms. - -# This is the Nim version used locally and in regular CI builds. -# Can be a specific version tag, a branch name, or a commit hash. -# Can be overridden by setting the NIM_COMMIT environment variable -# before calling make. -# -# For readability in CI, if NIM_COMMIT is set to "pinned", -# this will also default to the version pinned here. -# -# If NIM_COMMIT is set to "nimbusbuild", this will use the -# version pinned by nimbus-build-system. -PINNED_NIM_VERSION := v2.2.4 - -ifeq ($(NIM_COMMIT),) -NIM_COMMIT := $(PINNED_NIM_VERSION) -else ifeq ($(NIM_COMMIT),pinned) -NIM_COMMIT := $(PINNED_NIM_VERSION) -endif - -ifeq ($(NIM_COMMIT),nimbusbuild) -undefine NIM_COMMIT -else -export NIM_COMMIT -endif - -SHELL := bash # the shell used internally by Make - -# used inside the included makefiles -BUILD_SYSTEM_DIR := vendor/nimbus-build-system - -# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics -# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker -DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure - -LINK_PCRE := 0 - -ifeq ($(OS),Windows_NT) - ifeq ($(PROCESSOR_ARCHITECTURE), AMD64) - ARCH = x86_64 - endif - ifeq ($(PROCESSOR_ARCHITECTURE), ARM64) - ARCH = arm64 - endif -else - UNAME_P := $(shell uname -m) - ifneq ($(filter $(UNAME_P), i686 i386 x86_64),) - ARCH = x86_64 - endif - ifneq ($(filter $(UNAME_P), aarch64 arm),) - ARCH = arm64 - endif -endif - -ifeq ($(ARCH), x86_64) - CXXFLAGS ?= -std=c++17 -mssse3 -else - CXXFLAGS ?= -std=c++17 -endif -export CXXFLAGS - -# we don't want an error here, so we can handle things later, in the ".DEFAULT" target --include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk - -.PHONY: \ - all \ - clean \ - coverage \ - deps \ - libbacktrace \ - test \ - update - -ifeq ($(NIM_PARAMS),) -# "variables.mk" was not included, so we update the submodules. -GIT_SUBMODULE_UPDATE := git submodule update --init --recursive -.DEFAULT: - +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ - $(GIT_SUBMODULE_UPDATE); \ - echo -# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: -# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles -# -# After restarting, it will execute its original goal, so we don't have to start a child Make here -# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? - -else # "variables.mk" was included. Business as usual until the end of this file. - -# default target, because it's the first one that doesn't start with '.' - -# Builds the crawler binary -all: | build deps - echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims - -# must be included after the default target --include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk - -# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims -ifeq ($(USE_LIBBACKTRACE), 0) -NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace -else -NIM_PARAMS := $(NIM_PARAMS) -d:release -endif - -deps: | deps-common nat-libs -ifneq ($(USE_LIBBACKTRACE), 0) -deps: | libbacktrace -endif - -update: | update-common - -# detecting the os -ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... - detected_OS := Windows -else ifeq ($(strip $(shell uname)),Darwin) - detected_OS := macOS -else - # e.g. Linux - detected_OS := $(strip $(shell uname)) -endif - -# Builds and run a part of the test suite -test: | build deps - echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims - -# nim-libbacktrace -LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 -libbacktrace: -ifeq ($(detected_OS), Windows) -# MSYS2 detection -ifneq ($(MSYSTEM),) - + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'" -else - + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) -endif -else - + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) -endif - -# usual cleaning -clean: | clean-common - rm -rf build -ifneq ($(USE_LIBBACKTRACE), 0) - + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) -endif - -############ -## Format ## -############ -.PHONY: build-nph install-nph-hook clean-nph print-nph-path - -# Default location for nph binary shall be next to nim binary to make it available on the path. -NPH:=$(shell dirname $(NIM_BINARY))/nph - -build-nph: -ifeq ("$(wildcard $(NPH))","") - $(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \ - mv vendor/nph/src/nph $(shell dirname $(NPH)) - echo "nph utility is available at " $(NPH) -endif - -GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit - -install-nph-hook: build-nph -ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","") - cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK) -else - echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override" - exit 1 -endif - -nph/%: build-nph - echo -e $(FORMAT_MSG) "nph/$*" && \ - $(NPH) $* - -format: - $(NPH) *.nim - $(NPH) codexcrawler/ - $(NPH) tests/ - -clean-nph: - rm -f $(NPH) - -# To avoid hardcoding nph binary location in several places -print-nph-path: - echo "$(NPH)" - -clean: | clean-nph - -endif # "variables.mk" was not included +# Copyright (c) 2020 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# This is the Nim version used locally and in regular CI builds. +# Can be a specific version tag, a branch name, or a commit hash. +# Can be overridden by setting the NIM_COMMIT environment variable +# before calling make. +# +# For readability in CI, if NIM_COMMIT is set to "pinned", +# this will also default to the version pinned here. +# +# If NIM_COMMIT is set to "nimbusbuild", this will use the +# version pinned by nimbus-build-system. +PINNED_NIM_VERSION := v2.2.4 + +ifeq ($(NIM_COMMIT),) +NIM_COMMIT := $(PINNED_NIM_VERSION) +else ifeq ($(NIM_COMMIT),pinned) +NIM_COMMIT := $(PINNED_NIM_VERSION) +endif + +ifeq ($(NIM_COMMIT),nimbusbuild) +undefine NIM_COMMIT +else +export NIM_COMMIT +endif + +SHELL := bash # the shell used internally by Make + +# used inside the included makefiles +BUILD_SYSTEM_DIR := vendor/nimbus-build-system + +# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics +# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker +DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure + +LINK_PCRE := 0 + +ifeq ($(OS),Windows_NT) + ifeq ($(PROCESSOR_ARCHITECTURE), AMD64) + ARCH = x86_64 + endif + ifeq ($(PROCESSOR_ARCHITECTURE), ARM64) + ARCH = arm64 + endif +else + UNAME_P := $(shell uname -m) + ifneq ($(filter $(UNAME_P), i686 i386 x86_64),) + ARCH = x86_64 + endif + ifneq ($(filter $(UNAME_P), aarch64 arm),) + ARCH = arm64 + endif +endif + +ifeq ($(ARCH), x86_64) + CXXFLAGS ?= -std=c++17 -mssse3 +else + CXXFLAGS ?= -std=c++17 +endif +export CXXFLAGS + +# we don't want an error here, so we can handle things later, in the ".DEFAULT" target +-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk + +.PHONY: \ + all \ + clean \ + coverage \ + deps \ + libbacktrace \ + test \ + update + +ifeq ($(NIM_PARAMS),) +# "variables.mk" was not included, so we update the submodules. +GIT_SUBMODULE_UPDATE := git submodule update --init --recursive +.DEFAULT: + +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ + $(GIT_SUBMODULE_UPDATE); \ + echo +# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: +# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles +# +# After restarting, it will execute its original goal, so we don't have to start a child Make here +# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? + +else # "variables.mk" was included. Business as usual until the end of this file. + +# default target, because it's the first one that doesn't start with '.' + +# Builds the crawler binary +all: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims + +# must be included after the default target +-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk + +# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims +ifeq ($(USE_LIBBACKTRACE), 0) +NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace +else +NIM_PARAMS := $(NIM_PARAMS) -d:release +endif + +deps: | deps-common nat-libs +ifneq ($(USE_LIBBACKTRACE), 0) +deps: | libbacktrace +endif + +update: | update-common + +# detecting the os +ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... + detected_OS := Windows +else ifeq ($(strip $(shell uname)),Darwin) + detected_OS := macOS +else + # e.g. Linux + detected_OS := $(strip $(shell uname)) +endif + +# Builds and run a part of the test suite +test: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims + +# nim-libbacktrace +LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 +libbacktrace: +ifeq ($(detected_OS), Windows) +# MSYS2 detection +ifneq ($(MSYSTEM),) + + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'" +else + + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) +endif +else + + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) +endif + +# usual cleaning +clean: | clean-common + rm -rf build +ifneq ($(USE_LIBBACKTRACE), 0) + + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) +endif + +############ +## Format ## +############ +.PHONY: build-nph install-nph-hook clean-nph print-nph-path + +# Default location for nph binary shall be next to nim binary to make it available on the path. +NPH:=$(shell dirname $(NIM_BINARY))/nph + +build-nph: +ifeq ("$(wildcard $(NPH))","") + $(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \ + mv vendor/nph/src/nph $(shell dirname $(NPH)) + echo "nph utility is available at " $(NPH) +endif + +GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit + +install-nph-hook: build-nph +ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","") + cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK) +else + echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override" + exit 1 +endif + +nph/%: build-nph + echo -e $(FORMAT_MSG) "nph/$*" && \ + $(NPH) $* + +format: + $(NPH) *.nim + $(NPH) codexcrawler/ + $(NPH) tests/ + +clean-nph: + rm -f $(NPH) + +# To avoid hardcoding nph binary location in several places +print-nph-path: + echo "$(NPH)" + +clean: | clean-nph + +endif # "variables.mk" was not included diff --git a/README.md b/README.md index 68948bb..08fa271 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ -# Codex Network Crawler - -![Crawler](crawler.png) - -[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) -[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](#stability) -[![CI (GitHub Actions)](https://github.com/codex-storage/nim-codex-dht/workflows/CI/badge.svg?branch=master)](https://github.com/codex-storage/nim-codex-dht/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amaster) -[![codecov](https://codecov.io/gh/codex-storage/nim-codex-dht/branch/master/graph/badge.svg?token=tlmMJgU4l7)](https://codecov.io/gh/codex-storage/nim-codex-dht) - -# !! Work in Progress !! - -This project uses nim-codex-dht, nim-libp2p, nim-ethers, and nim-metrics to create a metrics service. The crawler will traverse the Codex network and produce metrics such as: -- Number of DHT nodes (alive vs total) -- P2P connectivity (percentage) -- Storage contract statistics (created, total size, average size, average duration, pricing information??) - -Metrics are published from a scrape target. - -# Usage - -```sh -nimble format -nimble build -nimble test -nimble run -``` +# Codex Network Crawler + +![Crawler](crawler.png) + +[![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) +[![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg)](#stability) +[![CI (GitHub Actions)](https://github.com/codex-storage/nim-codex-dht/workflows/CI/badge.svg?branch=master)](https://github.com/codex-storage/nim-codex-dht/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amaster) +[![codecov](https://codecov.io/gh/codex-storage/nim-codex-dht/branch/master/graph/badge.svg?token=tlmMJgU4l7)](https://codecov.io/gh/codex-storage/nim-codex-dht) + +# !! Work in Progress !! + +This project uses nim-codex-dht, nim-libp2p, nim-ethers, and nim-metrics to create a metrics service. The crawler will traverse the Codex network and produce metrics such as: +- Number of DHT nodes (alive vs total) +- P2P connectivity (percentage) +- Storage contract statistics (created, total size, average size, average duration, pricing information??) + +Metrics are published from a scrape target. + +# Usage + +```sh +nimble format +nimble build +nimble test +nimble run +``` diff --git a/codexcrawler.nimble b/codexcrawler.nimble index c14c124..6eb3c19 100644 --- a/codexcrawler.nimble +++ b/codexcrawler.nimble @@ -1,39 +1,39 @@ -# Package - -version = "0.0.1" -author = "Codex core contributors" -description = "Crawler for Codex networks" -license = "MIT" -skipDirs = @["tests"] -bin = @["codexcrawler"] -binDir = "build" - -# Dependencies -requires "nim >= 2.0.14 & < 3.0.0" -requires "secp256k1 >= 0.6.0 & < 0.7.0" -requires "protobuf_serialization >= 0.3.0 & < 0.4.0" -requires "nimcrypto >= 0.6.2 & < 0.7.0" -requires "bearssl >= 0.2.5 & < 0.3.0" -requires "chronicles >= 0.10.2 & < 0.11.0" -requires "chronos >= 4.0.3 & < 4.1.0" -# requires "libp2p >= 1.5.0 & < 2.0.0" -requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses. -requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3" - -requires "metrics >= 0.1.0 & < 0.2.0" -requires "stew >= 0.2.0 & < 0.3.0" -requires "stint >= 0.8.1 & < 0.9.0" -requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0" -requires "questionable >= 0.10.15 & < 0.11.0" -requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59" -requires "docopt >= 0.7.1 & < 1.0.0" -requires "nph >= 0.6.1 & < 1.0.0" -requires "ethers >= 1.0.0 & < 2.0.0" - -task format, "Formatting...": - exec "nph ./" - -task test, "Run tests": - exec "nimble install -d -y" - withDir "tests": - exec "nimble test" +# Package + +version = "0.0.1" +author = "Codex core contributors" +description = "Crawler for Codex networks" +license = "MIT" +skipDirs = @["tests"] +bin = @["codexcrawler"] +binDir = "build" + +# Dependencies +requires "nim >= 2.0.14 & < 3.0.0" +requires "secp256k1 >= 0.6.0 & < 0.7.0" +requires "protobuf_serialization >= 0.3.0 & < 0.4.0" +requires "nimcrypto >= 0.6.2 & < 0.7.0" +requires "bearssl >= 0.2.5 & < 0.3.0" +requires "chronicles >= 0.10.2 & < 0.11.0" +requires "chronos >= 4.0.3 & < 4.1.0" +# requires "libp2p >= 1.5.0 & < 2.0.0" +requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses. +requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3" + +requires "metrics >= 0.1.0 & < 0.2.0" +requires "stew >= 0.2.0 & < 0.3.0" +requires "stint >= 0.8.1 & < 0.9.0" +requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0" +requires "questionable >= 0.10.15 & < 0.11.0" +requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59" +requires "docopt >= 0.7.1 & < 1.0.0" +requires "nph >= 0.6.1 & < 1.0.0" +requires "ethers >= 1.0.0 & < 2.0.0" + +task format, "Formatting...": + exec "nph ./" + +task test, "Run tests": + exec "nimble install -d -y" + withDir "tests": + exec "nimble test" diff --git a/codexcrawler/config.nim b/codexcrawler/config.nim index ef20ae2..6b21207 100644 --- a/codexcrawler/config.nim +++ b/codexcrawler/config.nim @@ -6,31 +6,31 @@ import pkg/codexdht import ./utils/version let doc = - """ -Codex Network Crawler. Generates network metrics. - -Usage: - codexcrawler [--logLevel=] [--publicIp=] [--metricsAddress=] [--metricsPort=

] [--dataDir=

] [--discoveryPort=

] [--bootNodes=] [--dhtEnable=] [--stepDelay=] [--revisitDelay=] [--checkDelay=] [--expiryDelay=] [--marketplaceEnable=] [--ethProvider=] [--marketplaceAddress=] [--requestCheckDelay=] - -Options: - --logLevel= Sets log level [default: INFO] - --publicIp= Public IP address where this instance is reachable. - --metricsAddress= Listen address of the metrics server [default: 0.0.0.0] - --metricsPort=

Listen HTTP port of the metrics server [default: 8008] - --dataDir=

Directory for storing data [default: crawler_data] - --discoveryPort=

Port used for DHT [default: 8090] - --bootNodes= Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs] - - --dhtEnable= Set to "1" to enable DHT crawler [default: 1] - --stepDelay= Delay in milliseconds per node visit [default: 1000] - --revisitDelay= Delay in minutes after which a node can be revisited [default: 60] - --checkDelay= Delay with which the 'revisitDelay' is checked for all known nodes [default: 10] - --expiryDelay= Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h) - - --marketplaceEnable= Set to "1" to enable marketplace metrics [default: 1] - --ethProvider= Address including http(s) or ws of the eth provider - --marketplaceAddress= Eth address of Codex contracts deployment - --requestCheckDelay= Delay in minutes after which storage contract status is (re)checked [default: 10] + """ +Codex Network Crawler. Generates network metrics. + +Usage: + codexcrawler [--logLevel=] [--publicIp=] [--metricsAddress=] [--metricsPort=

] [--dataDir=

] [--discoveryPort=

] [--bootNodes=] [--dhtEnable=] [--stepDelay=] [--revisitDelay=] [--checkDelay=] [--expiryDelay=] [--marketplaceEnable=] [--ethProvider=] [--marketplaceAddress=] [--requestCheckDelay=] + +Options: + --logLevel= Sets log level [default: INFO] + --publicIp= Public IP address where this instance is reachable. + --metricsAddress= Listen address of the metrics server [default: 0.0.0.0] + --metricsPort=

Listen HTTP port of the metrics server [default: 8008] + --dataDir=

Directory for storing data [default: crawler_data] + --discoveryPort=

Port used for DHT [default: 8090] + --bootNodes= Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs] + + --dhtEnable= Set to "1" to enable DHT crawler [default: 1] + --stepDelay= Delay in milliseconds per node visit [default: 1000] + --revisitDelay= Delay in minutes after which a node can be revisited [default: 60] + --checkDelay= Delay with which the 'revisitDelay' is checked for all known nodes [default: 10] + --expiryDelay= Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h) + + --marketplaceEnable= Set to "1" to enable marketplace metrics [default: 1] + --ethProvider= Address including http(s) or ws of the eth provider + --marketplaceAddress= Eth address of Codex contracts deployment + --requestCheckDelay= Delay in minutes after which storage contract status is (re)checked [default: 10] """ import strutils diff --git a/codexcrawler/services/marketplace/readme.md b/codexcrawler/services/marketplace/readme.md index fd2b28a..bf693cc 100644 --- a/codexcrawler/services/marketplace/readme.md +++ b/codexcrawler/services/marketplace/readme.md @@ -1,2 +1,2 @@ -These are copied from nim-codex v0.2.0. -There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled. +These are copied from nim-codex v0.2.0. +There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled. diff --git a/docker/crawler.Dockerfile b/docker/crawler.Dockerfile index 0d68a08..1d453be 100644 --- a/docker/crawler.Dockerfile +++ b/docker/crawler.Dockerfile @@ -1,36 +1,36 @@ -# Variables -ARG BUILDER=ubuntu:24.04 -ARG IMAGE=${BUILDER} -ARG BUILD_HOME=/src -ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4} -ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}" -ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1} -ARG APP_HOME=/crawler - -# Build -FROM ${BUILDER} AS builder -ARG BUILD_HOME -ARG MAKE_PARALLEL -ARG NIMFLAGS -ARG USE_LIBBACKTRACE -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update && apt-get install -y git cmake curl make bash build-essential - -WORKDIR ${BUILD_HOME} -COPY . . -RUN make -j ${MAKE_PARALLEL} update -RUN make -j ${MAKE_PARALLEL} - -# Create -FROM ${IMAGE} -ARG BUILD_HOME -ARG APP_HOME - -WORKDIR ${APP_HOME} -COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin -COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh / -RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/* - -ENTRYPOINT ["/docker-entrypoint.sh"] -CMD ["codexcrawler"] +# Variables +ARG BUILDER=ubuntu:24.04 +ARG IMAGE=${BUILDER} +ARG BUILD_HOME=/src +ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4} +ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}" +ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1} +ARG APP_HOME=/crawler + +# Build +FROM ${BUILDER} AS builder +ARG BUILD_HOME +ARG MAKE_PARALLEL +ARG NIMFLAGS +ARG USE_LIBBACKTRACE +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y git cmake curl make bash build-essential + +WORKDIR ${BUILD_HOME} +COPY . . +RUN make -j ${MAKE_PARALLEL} update +RUN make -j ${MAKE_PARALLEL} + +# Create +FROM ${IMAGE} +ARG BUILD_HOME +ARG APP_HOME + +WORKDIR ${APP_HOME} +COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin +COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh / +RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/* + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["codexcrawler"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ebc8c12..71465db 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -1,20 +1,20 @@ -services: - crawler1: - build: - context: .. - dockerfile: ./docker/crawler.Dockerfile - environment: - - CRAWLER_LOGLEVEL=TRACE - # - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage - - CRAWLER_METRICSADDRESS=0.0.0.0 - - CRAWLER_METRICSPORT=8008 - - CRAWLER_DATADIR=crawler_data - - CRAWLER_DISCPORT=8090 - - CRAWLER_BOOTNODES=testnet_sprs - - CRAWLER_STEPDELAY=3000 - - CRAWLER_REVISITDELAY=1440 - ports: - - 8008:8008/tcp # Metrics - - 8090:8090/udp # DHT discovery - volumes: - - ./crawler_data:/crawler_data:z +services: + crawler1: + build: + context: .. + dockerfile: ./docker/crawler.Dockerfile + environment: + - CRAWLER_LOGLEVEL=TRACE + # - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage + - CRAWLER_METRICSADDRESS=0.0.0.0 + - CRAWLER_METRICSPORT=8008 + - CRAWLER_DATADIR=crawler_data + - CRAWLER_DISCPORT=8090 + - CRAWLER_BOOTNODES=testnet_sprs + - CRAWLER_STEPDELAY=3000 + - CRAWLER_REVISITDELAY=1440 + ports: + - 8008:8008/tcp # Metrics + - 8090:8090/udp # DHT discovery + volumes: + - ./crawler_data:/crawler_data:z diff --git a/env.sh b/env.sh index 4deb2f1..697a426 100755 --- a/env.sh +++ b/env.sh @@ -1,7 +1,7 @@ -#!/usr/bin/env bash - -# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file -# and we fall back to a Zsh-specific special var to also support Zsh. -REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" -ABS_PATH="$(cd ${REL_PATH}; pwd)" -source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh +#!/usr/bin/env bash + +# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file +# and we fall back to a Zsh-specific special var to also support Zsh. +REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})" +ABS_PATH="$(cd ${REL_PATH}; pwd)" +source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh