mirror of
https://github.com/logos-storage/logos-storage-network-crawler.git
synced 2026-01-02 13:33:08 +00:00
adjusting line endings
This commit is contained in:
thatben
parent
59b4b25667
commit
d71e1f1906
@ -1,6 +1,6 @@
|
|||||||
.github
|
.github
|
||||||
build
|
build
|
||||||
docs
|
docs
|
||||||
metrics
|
metrics
|
||||||
nimcache
|
nimcache
|
||||||
tests
|
tests
|
||||||
|
|||||||
2
.gitattributes
vendored
Normal file
2
.gitattributes
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
env.sh text eol=lf
|
||||||
|
docker/docker-entrypoint.sh text eol=lf
|
||||||
42
.github/workflows/docker.yml
vendored
42
.github/workflows/docker.yml
vendored
@ -1,21 +1,21 @@
|
|||||||
name: Docker
|
name: Docker
|
||||||
|
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- master
|
- master
|
||||||
tags:
|
tags:
|
||||||
- 'v*.*.*'
|
- 'v*.*.*'
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-and-push:
|
build-and-push:
|
||||||
name: Build and Push
|
name: Build and Push
|
||||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||||
with:
|
with:
|
||||||
docker_file: docker/crawler.Dockerfile
|
docker_file: docker/crawler.Dockerfile
|
||||||
dockerhub_repo: codexstorage/codex-network-crawler
|
dockerhub_repo: codexstorage/codex-network-crawler
|
||||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|||||||
86
.github/workflows/main.yml
vendored
86
.github/workflows/main.yml
vendored
@ -1,43 +1,43 @@
|
|||||||
name: CI
|
name: CI
|
||||||
|
|
||||||
on: [push, pull_request]
|
on: [push, pull_request]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
linting:
|
linting:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Check `nph` formatting
|
- name: Check `nph` formatting
|
||||||
uses: arnetheduck/nph-action@v1
|
uses: arnetheduck/nph-action@v1
|
||||||
with:
|
with:
|
||||||
version: 0.6.1
|
version: 0.6.1
|
||||||
options: "./"
|
options: "./"
|
||||||
fail: true
|
fail: true
|
||||||
suggest: true
|
suggest: true
|
||||||
|
|
||||||
test:
|
test:
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
os: [ubuntu-latest, macOS-latest, windows-latest]
|
os: [ubuntu-latest, macOS-latest, windows-latest]
|
||||||
nim: [2.2.4]
|
nim: [2.2.4]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
submodules: true
|
submodules: true
|
||||||
|
|
||||||
- name: Build Nim and dependencies
|
- name: Build Nim and dependencies
|
||||||
run: |
|
run: |
|
||||||
which gcc
|
which gcc
|
||||||
gcc --version
|
gcc --version
|
||||||
make update
|
make update
|
||||||
./env.sh nim --version
|
./env.sh nim --version
|
||||||
make
|
make
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
run: |
|
run: |
|
||||||
make test
|
make test
|
||||||
|
|
||||||
- name: Build crawler
|
- name: Build crawler
|
||||||
run: |
|
run: |
|
||||||
make
|
make
|
||||||
|
|||||||
16
.gitignore
vendored
16
.gitignore
vendored
@ -1,9 +1,9 @@
|
|||||||
*
|
*
|
||||||
!*.*
|
!*.*
|
||||||
!*/
|
!*/
|
||||||
nimble.paths
|
nimble.paths
|
||||||
nimbus-build-system.paths
|
nimbus-build-system.paths
|
||||||
vendor/*
|
vendor/*
|
||||||
*.exe
|
*.exe
|
||||||
crawler_data
|
crawler_data
|
||||||
.update.timestamp
|
.update.timestamp
|
||||||
|
|||||||
394
Makefile
394
Makefile
@ -1,197 +1,197 @@
|
|||||||
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
|
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
|
||||||
# either of:
|
# either of:
|
||||||
# - Apache License, version 2.0
|
# - Apache License, version 2.0
|
||||||
# - MIT license
|
# - MIT license
|
||||||
# at your option. This file may not be copied, modified, or distributed except
|
# at your option. This file may not be copied, modified, or distributed except
|
||||||
# according to those terms.
|
# according to those terms.
|
||||||
|
|
||||||
# This is the Nim version used locally and in regular CI builds.
|
# This is the Nim version used locally and in regular CI builds.
|
||||||
# Can be a specific version tag, a branch name, or a commit hash.
|
# Can be a specific version tag, a branch name, or a commit hash.
|
||||||
# Can be overridden by setting the NIM_COMMIT environment variable
|
# Can be overridden by setting the NIM_COMMIT environment variable
|
||||||
# before calling make.
|
# before calling make.
|
||||||
#
|
#
|
||||||
# For readability in CI, if NIM_COMMIT is set to "pinned",
|
# For readability in CI, if NIM_COMMIT is set to "pinned",
|
||||||
# this will also default to the version pinned here.
|
# this will also default to the version pinned here.
|
||||||
#
|
#
|
||||||
# If NIM_COMMIT is set to "nimbusbuild", this will use the
|
# If NIM_COMMIT is set to "nimbusbuild", this will use the
|
||||||
# version pinned by nimbus-build-system.
|
# version pinned by nimbus-build-system.
|
||||||
PINNED_NIM_VERSION := v2.2.4
|
PINNED_NIM_VERSION := v2.2.4
|
||||||
|
|
||||||
ifeq ($(NIM_COMMIT),)
|
ifeq ($(NIM_COMMIT),)
|
||||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||||
else ifeq ($(NIM_COMMIT),pinned)
|
else ifeq ($(NIM_COMMIT),pinned)
|
||||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ifeq ($(NIM_COMMIT),nimbusbuild)
|
ifeq ($(NIM_COMMIT),nimbusbuild)
|
||||||
undefine NIM_COMMIT
|
undefine NIM_COMMIT
|
||||||
else
|
else
|
||||||
export NIM_COMMIT
|
export NIM_COMMIT
|
||||||
endif
|
endif
|
||||||
|
|
||||||
SHELL := bash # the shell used internally by Make
|
SHELL := bash # the shell used internally by Make
|
||||||
|
|
||||||
# used inside the included makefiles
|
# used inside the included makefiles
|
||||||
BUILD_SYSTEM_DIR := vendor/nimbus-build-system
|
BUILD_SYSTEM_DIR := vendor/nimbus-build-system
|
||||||
|
|
||||||
# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
|
# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
|
||||||
# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
|
# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
|
||||||
DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
|
DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
|
||||||
|
|
||||||
LINK_PCRE := 0
|
LINK_PCRE := 0
|
||||||
|
|
||||||
ifeq ($(OS),Windows_NT)
|
ifeq ($(OS),Windows_NT)
|
||||||
ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
|
ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
|
||||||
ARCH = x86_64
|
ARCH = x86_64
|
||||||
endif
|
endif
|
||||||
ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
|
ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
|
||||||
ARCH = arm64
|
ARCH = arm64
|
||||||
endif
|
endif
|
||||||
else
|
else
|
||||||
UNAME_P := $(shell uname -m)
|
UNAME_P := $(shell uname -m)
|
||||||
ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
|
ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
|
||||||
ARCH = x86_64
|
ARCH = x86_64
|
||||||
endif
|
endif
|
||||||
ifneq ($(filter $(UNAME_P), aarch64 arm),)
|
ifneq ($(filter $(UNAME_P), aarch64 arm),)
|
||||||
ARCH = arm64
|
ARCH = arm64
|
||||||
endif
|
endif
|
||||||
endif
|
endif
|
||||||
|
|
||||||
ifeq ($(ARCH), x86_64)
|
ifeq ($(ARCH), x86_64)
|
||||||
CXXFLAGS ?= -std=c++17 -mssse3
|
CXXFLAGS ?= -std=c++17 -mssse3
|
||||||
else
|
else
|
||||||
CXXFLAGS ?= -std=c++17
|
CXXFLAGS ?= -std=c++17
|
||||||
endif
|
endif
|
||||||
export CXXFLAGS
|
export CXXFLAGS
|
||||||
|
|
||||||
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
|
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
|
||||||
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
|
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
|
||||||
|
|
||||||
.PHONY: \
|
.PHONY: \
|
||||||
all \
|
all \
|
||||||
clean \
|
clean \
|
||||||
coverage \
|
coverage \
|
||||||
deps \
|
deps \
|
||||||
libbacktrace \
|
libbacktrace \
|
||||||
test \
|
test \
|
||||||
update
|
update
|
||||||
|
|
||||||
ifeq ($(NIM_PARAMS),)
|
ifeq ($(NIM_PARAMS),)
|
||||||
# "variables.mk" was not included, so we update the submodules.
|
# "variables.mk" was not included, so we update the submodules.
|
||||||
GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
|
GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
|
||||||
.DEFAULT:
|
.DEFAULT:
|
||||||
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
|
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
|
||||||
$(GIT_SUBMODULE_UPDATE); \
|
$(GIT_SUBMODULE_UPDATE); \
|
||||||
echo
|
echo
|
||||||
# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
|
# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
|
||||||
# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
|
# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
|
||||||
#
|
#
|
||||||
# After restarting, it will execute its original goal, so we don't have to start a child Make here
|
# After restarting, it will execute its original goal, so we don't have to start a child Make here
|
||||||
# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
|
# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
|
||||||
|
|
||||||
else # "variables.mk" was included. Business as usual until the end of this file.
|
else # "variables.mk" was included. Business as usual until the end of this file.
|
||||||
|
|
||||||
# default target, because it's the first one that doesn't start with '.'
|
# default target, because it's the first one that doesn't start with '.'
|
||||||
|
|
||||||
# Builds the crawler binary
|
# Builds the crawler binary
|
||||||
all: | build deps
|
all: | build deps
|
||||||
echo -e $(BUILD_MSG) "build/$@" && \
|
echo -e $(BUILD_MSG) "build/$@" && \
|
||||||
$(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims
|
$(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims
|
||||||
|
|
||||||
# must be included after the default target
|
# must be included after the default target
|
||||||
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
|
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
|
||||||
|
|
||||||
# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
|
# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
|
||||||
ifeq ($(USE_LIBBACKTRACE), 0)
|
ifeq ($(USE_LIBBACKTRACE), 0)
|
||||||
NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
|
NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
|
||||||
else
|
else
|
||||||
NIM_PARAMS := $(NIM_PARAMS) -d:release
|
NIM_PARAMS := $(NIM_PARAMS) -d:release
|
||||||
endif
|
endif
|
||||||
|
|
||||||
deps: | deps-common nat-libs
|
deps: | deps-common nat-libs
|
||||||
ifneq ($(USE_LIBBACKTRACE), 0)
|
ifneq ($(USE_LIBBACKTRACE), 0)
|
||||||
deps: | libbacktrace
|
deps: | libbacktrace
|
||||||
endif
|
endif
|
||||||
|
|
||||||
update: | update-common
|
update: | update-common
|
||||||
|
|
||||||
# detecting the os
|
# detecting the os
|
||||||
ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
|
ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
|
||||||
detected_OS := Windows
|
detected_OS := Windows
|
||||||
else ifeq ($(strip $(shell uname)),Darwin)
|
else ifeq ($(strip $(shell uname)),Darwin)
|
||||||
detected_OS := macOS
|
detected_OS := macOS
|
||||||
else
|
else
|
||||||
# e.g. Linux
|
# e.g. Linux
|
||||||
detected_OS := $(strip $(shell uname))
|
detected_OS := $(strip $(shell uname))
|
||||||
endif
|
endif
|
||||||
|
|
||||||
# Builds and run a part of the test suite
|
# Builds and run a part of the test suite
|
||||||
test: | build deps
|
test: | build deps
|
||||||
echo -e $(BUILD_MSG) "build/$@" && \
|
echo -e $(BUILD_MSG) "build/$@" && \
|
||||||
$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
|
$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
|
||||||
|
|
||||||
# nim-libbacktrace
|
# nim-libbacktrace
|
||||||
LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
|
LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
|
||||||
libbacktrace:
|
libbacktrace:
|
||||||
ifeq ($(detected_OS), Windows)
|
ifeq ($(detected_OS), Windows)
|
||||||
# MSYS2 detection
|
# MSYS2 detection
|
||||||
ifneq ($(MSYSTEM),)
|
ifneq ($(MSYSTEM),)
|
||||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'"
|
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'"
|
||||||
else
|
else
|
||||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
||||||
endif
|
endif
|
||||||
else
|
else
|
||||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
# usual cleaning
|
# usual cleaning
|
||||||
clean: | clean-common
|
clean: | clean-common
|
||||||
rm -rf build
|
rm -rf build
|
||||||
ifneq ($(USE_LIBBACKTRACE), 0)
|
ifneq ($(USE_LIBBACKTRACE), 0)
|
||||||
+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
|
+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
############
|
############
|
||||||
## Format ##
|
## Format ##
|
||||||
############
|
############
|
||||||
.PHONY: build-nph install-nph-hook clean-nph print-nph-path
|
.PHONY: build-nph install-nph-hook clean-nph print-nph-path
|
||||||
|
|
||||||
# Default location for nph binary shall be next to nim binary to make it available on the path.
|
# Default location for nph binary shall be next to nim binary to make it available on the path.
|
||||||
NPH:=$(shell dirname $(NIM_BINARY))/nph
|
NPH:=$(shell dirname $(NIM_BINARY))/nph
|
||||||
|
|
||||||
build-nph:
|
build-nph:
|
||||||
ifeq ("$(wildcard $(NPH))","")
|
ifeq ("$(wildcard $(NPH))","")
|
||||||
$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
|
$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
|
||||||
mv vendor/nph/src/nph $(shell dirname $(NPH))
|
mv vendor/nph/src/nph $(shell dirname $(NPH))
|
||||||
echo "nph utility is available at " $(NPH)
|
echo "nph utility is available at " $(NPH)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
|
GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
|
||||||
|
|
||||||
install-nph-hook: build-nph
|
install-nph-hook: build-nph
|
||||||
ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
|
ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
|
||||||
cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
|
cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
|
||||||
else
|
else
|
||||||
echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
|
echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
|
||||||
exit 1
|
exit 1
|
||||||
endif
|
endif
|
||||||
|
|
||||||
nph/%: build-nph
|
nph/%: build-nph
|
||||||
echo -e $(FORMAT_MSG) "nph/$*" && \
|
echo -e $(FORMAT_MSG) "nph/$*" && \
|
||||||
$(NPH) $*
|
$(NPH) $*
|
||||||
|
|
||||||
format:
|
format:
|
||||||
$(NPH) *.nim
|
$(NPH) *.nim
|
||||||
$(NPH) codexcrawler/
|
$(NPH) codexcrawler/
|
||||||
$(NPH) tests/
|
$(NPH) tests/
|
||||||
|
|
||||||
clean-nph:
|
clean-nph:
|
||||||
rm -f $(NPH)
|
rm -f $(NPH)
|
||||||
|
|
||||||
# To avoid hardcoding nph binary location in several places
|
# To avoid hardcoding nph binary location in several places
|
||||||
print-nph-path:
|
print-nph-path:
|
||||||
echo "$(NPH)"
|
echo "$(NPH)"
|
||||||
|
|
||||||
clean: | clean-nph
|
clean: | clean-nph
|
||||||
|
|
||||||
endif # "variables.mk" was not included
|
endif # "variables.mk" was not included
|
||||||
|
|||||||
54
README.md
54
README.md
@ -1,27 +1,27 @@
|
|||||||
# Codex Network Crawler
|
# Codex Network Crawler
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
[](https://opensource.org/licenses/Apache-2.0)
|
[](https://opensource.org/licenses/Apache-2.0)
|
||||||
[](https://opensource.org/licenses/MIT)
|
[](https://opensource.org/licenses/MIT)
|
||||||
[](#stability)
|
[](#stability)
|
||||||
[](https://github.com/codex-storage/nim-codex-dht/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amaster)
|
[](https://github.com/codex-storage/nim-codex-dht/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amaster)
|
||||||
[](https://codecov.io/gh/codex-storage/nim-codex-dht)
|
[](https://codecov.io/gh/codex-storage/nim-codex-dht)
|
||||||
|
|
||||||
# !! Work in Progress !!
|
# !! Work in Progress !!
|
||||||
|
|
||||||
This project uses nim-codex-dht, nim-libp2p, nim-ethers, and nim-metrics to create a metrics service. The crawler will traverse the Codex network and produce metrics such as:
|
This project uses nim-codex-dht, nim-libp2p, nim-ethers, and nim-metrics to create a metrics service. The crawler will traverse the Codex network and produce metrics such as:
|
||||||
- Number of DHT nodes (alive vs total)
|
- Number of DHT nodes (alive vs total)
|
||||||
- P2P connectivity (percentage)
|
- P2P connectivity (percentage)
|
||||||
- Storage contract statistics (created, total size, average size, average duration, pricing information??)
|
- Storage contract statistics (created, total size, average size, average duration, pricing information??)
|
||||||
|
|
||||||
Metrics are published from a scrape target.
|
Metrics are published from a scrape target.
|
||||||
|
|
||||||
# Usage
|
# Usage
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
nimble format
|
nimble format
|
||||||
nimble build
|
nimble build
|
||||||
nimble test
|
nimble test
|
||||||
nimble run
|
nimble run
|
||||||
```
|
```
|
||||||
|
|||||||
@ -1,39 +1,39 @@
|
|||||||
# Package
|
# Package
|
||||||
|
|
||||||
version = "0.0.1"
|
version = "0.0.1"
|
||||||
author = "Codex core contributors"
|
author = "Codex core contributors"
|
||||||
description = "Crawler for Codex networks"
|
description = "Crawler for Codex networks"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
skipDirs = @["tests"]
|
skipDirs = @["tests"]
|
||||||
bin = @["codexcrawler"]
|
bin = @["codexcrawler"]
|
||||||
binDir = "build"
|
binDir = "build"
|
||||||
|
|
||||||
# Dependencies
|
# Dependencies
|
||||||
requires "nim >= 2.0.14 & < 3.0.0"
|
requires "nim >= 2.0.14 & < 3.0.0"
|
||||||
requires "secp256k1 >= 0.6.0 & < 0.7.0"
|
requires "secp256k1 >= 0.6.0 & < 0.7.0"
|
||||||
requires "protobuf_serialization >= 0.3.0 & < 0.4.0"
|
requires "protobuf_serialization >= 0.3.0 & < 0.4.0"
|
||||||
requires "nimcrypto >= 0.6.2 & < 0.7.0"
|
requires "nimcrypto >= 0.6.2 & < 0.7.0"
|
||||||
requires "bearssl >= 0.2.5 & < 0.3.0"
|
requires "bearssl >= 0.2.5 & < 0.3.0"
|
||||||
requires "chronicles >= 0.10.2 & < 0.11.0"
|
requires "chronicles >= 0.10.2 & < 0.11.0"
|
||||||
requires "chronos >= 4.0.3 & < 4.1.0"
|
requires "chronos >= 4.0.3 & < 4.1.0"
|
||||||
# requires "libp2p >= 1.5.0 & < 2.0.0"
|
# requires "libp2p >= 1.5.0 & < 2.0.0"
|
||||||
requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses.
|
requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses.
|
||||||
requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3"
|
requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3"
|
||||||
|
|
||||||
requires "metrics >= 0.1.0 & < 0.2.0"
|
requires "metrics >= 0.1.0 & < 0.2.0"
|
||||||
requires "stew >= 0.2.0 & < 0.3.0"
|
requires "stew >= 0.2.0 & < 0.3.0"
|
||||||
requires "stint >= 0.8.1 & < 0.9.0"
|
requires "stint >= 0.8.1 & < 0.9.0"
|
||||||
requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0"
|
requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0"
|
||||||
requires "questionable >= 0.10.15 & < 0.11.0"
|
requires "questionable >= 0.10.15 & < 0.11.0"
|
||||||
requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59"
|
requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59"
|
||||||
requires "docopt >= 0.7.1 & < 1.0.0"
|
requires "docopt >= 0.7.1 & < 1.0.0"
|
||||||
requires "nph >= 0.6.1 & < 1.0.0"
|
requires "nph >= 0.6.1 & < 1.0.0"
|
||||||
requires "ethers >= 1.0.0 & < 2.0.0"
|
requires "ethers >= 1.0.0 & < 2.0.0"
|
||||||
|
|
||||||
task format, "Formatting...":
|
task format, "Formatting...":
|
||||||
exec "nph ./"
|
exec "nph ./"
|
||||||
|
|
||||||
task test, "Run tests":
|
task test, "Run tests":
|
||||||
exec "nimble install -d -y"
|
exec "nimble install -d -y"
|
||||||
withDir "tests":
|
withDir "tests":
|
||||||
exec "nimble test"
|
exec "nimble test"
|
||||||
|
|||||||
@ -6,31 +6,31 @@ import pkg/codexdht
|
|||||||
import ./utils/version
|
import ./utils/version
|
||||||
|
|
||||||
let doc =
|
let doc =
|
||||||
"""
|
"""
|
||||||
Codex Network Crawler. Generates network metrics.
|
Codex Network Crawler. Generates network metrics.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>] [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>] [--requestCheckDelay=<m>]
|
codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>] [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>] [--requestCheckDelay=<m>]
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
--logLevel=<l> Sets log level [default: INFO]
|
--logLevel=<l> Sets log level [default: INFO]
|
||||||
--publicIp=<a> Public IP address where this instance is reachable.
|
--publicIp=<a> Public IP address where this instance is reachable.
|
||||||
--metricsAddress=<ip> Listen address of the metrics server [default: 0.0.0.0]
|
--metricsAddress=<ip> Listen address of the metrics server [default: 0.0.0.0]
|
||||||
--metricsPort=<p> Listen HTTP port of the metrics server [default: 8008]
|
--metricsPort=<p> Listen HTTP port of the metrics server [default: 8008]
|
||||||
--dataDir=<dir> Directory for storing data [default: crawler_data]
|
--dataDir=<dir> Directory for storing data [default: crawler_data]
|
||||||
--discoveryPort=<p> Port used for DHT [default: 8090]
|
--discoveryPort=<p> Port used for DHT [default: 8090]
|
||||||
--bootNodes=<n> Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
|
--bootNodes=<n> Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
|
||||||
|
|
||||||
--dhtEnable=<e> Set to "1" to enable DHT crawler [default: 1]
|
--dhtEnable=<e> Set to "1" to enable DHT crawler [default: 1]
|
||||||
--stepDelay=<ms> Delay in milliseconds per node visit [default: 1000]
|
--stepDelay=<ms> Delay in milliseconds per node visit [default: 1000]
|
||||||
--revisitDelay=<m> Delay in minutes after which a node can be revisited [default: 60]
|
--revisitDelay=<m> Delay in minutes after which a node can be revisited [default: 60]
|
||||||
--checkDelay=<m> Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
|
--checkDelay=<m> Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
|
||||||
--expiryDelay=<m> Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
|
--expiryDelay=<m> Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
|
||||||
|
|
||||||
--marketplaceEnable=<e> Set to "1" to enable marketplace metrics [default: 1]
|
--marketplaceEnable=<e> Set to "1" to enable marketplace metrics [default: 1]
|
||||||
--ethProvider=<a> Address including http(s) or ws of the eth provider
|
--ethProvider=<a> Address including http(s) or ws of the eth provider
|
||||||
--marketplaceAddress=<a> Eth address of Codex contracts deployment
|
--marketplaceAddress=<a> Eth address of Codex contracts deployment
|
||||||
--requestCheckDelay=<m> Delay in minutes after which storage contract status is (re)checked [default: 10]
|
--requestCheckDelay=<m> Delay in minutes after which storage contract status is (re)checked [default: 10]
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import strutils
|
import strutils
|
||||||
|
|||||||
@ -1,2 +1,2 @@
|
|||||||
These are copied from nim-codex v0.2.0.
|
These are copied from nim-codex v0.2.0.
|
||||||
There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled.
|
There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled.
|
||||||
|
|||||||
@ -1,36 +1,36 @@
|
|||||||
# Variables
|
# Variables
|
||||||
ARG BUILDER=ubuntu:24.04
|
ARG BUILDER=ubuntu:24.04
|
||||||
ARG IMAGE=${BUILDER}
|
ARG IMAGE=${BUILDER}
|
||||||
ARG BUILD_HOME=/src
|
ARG BUILD_HOME=/src
|
||||||
ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
|
ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
|
||||||
ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}"
|
ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}"
|
||||||
ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1}
|
ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1}
|
||||||
ARG APP_HOME=/crawler
|
ARG APP_HOME=/crawler
|
||||||
|
|
||||||
# Build
|
# Build
|
||||||
FROM ${BUILDER} AS builder
|
FROM ${BUILDER} AS builder
|
||||||
ARG BUILD_HOME
|
ARG BUILD_HOME
|
||||||
ARG MAKE_PARALLEL
|
ARG MAKE_PARALLEL
|
||||||
ARG NIMFLAGS
|
ARG NIMFLAGS
|
||||||
ARG USE_LIBBACKTRACE
|
ARG USE_LIBBACKTRACE
|
||||||
ENV DEBIAN_FRONTEND=noninteractive
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y git cmake curl make bash build-essential
|
RUN apt-get update && apt-get install -y git cmake curl make bash build-essential
|
||||||
|
|
||||||
WORKDIR ${BUILD_HOME}
|
WORKDIR ${BUILD_HOME}
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN make -j ${MAKE_PARALLEL} update
|
RUN make -j ${MAKE_PARALLEL} update
|
||||||
RUN make -j ${MAKE_PARALLEL}
|
RUN make -j ${MAKE_PARALLEL}
|
||||||
|
|
||||||
# Create
|
# Create
|
||||||
FROM ${IMAGE}
|
FROM ${IMAGE}
|
||||||
ARG BUILD_HOME
|
ARG BUILD_HOME
|
||||||
ARG APP_HOME
|
ARG APP_HOME
|
||||||
|
|
||||||
WORKDIR ${APP_HOME}
|
WORKDIR ${APP_HOME}
|
||||||
COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
|
COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
|
||||||
COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
|
COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
|
||||||
RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
|
RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||||
CMD ["codexcrawler"]
|
CMD ["codexcrawler"]
|
||||||
|
|||||||
@ -1,20 +1,20 @@
|
|||||||
services:
|
services:
|
||||||
crawler1:
|
crawler1:
|
||||||
build:
|
build:
|
||||||
context: ..
|
context: ..
|
||||||
dockerfile: ./docker/crawler.Dockerfile
|
dockerfile: ./docker/crawler.Dockerfile
|
||||||
environment:
|
environment:
|
||||||
- CRAWLER_LOGLEVEL=TRACE
|
- CRAWLER_LOGLEVEL=TRACE
|
||||||
# - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage
|
# - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage
|
||||||
- CRAWLER_METRICSADDRESS=0.0.0.0
|
- CRAWLER_METRICSADDRESS=0.0.0.0
|
||||||
- CRAWLER_METRICSPORT=8008
|
- CRAWLER_METRICSPORT=8008
|
||||||
- CRAWLER_DATADIR=crawler_data
|
- CRAWLER_DATADIR=crawler_data
|
||||||
- CRAWLER_DISCPORT=8090
|
- CRAWLER_DISCPORT=8090
|
||||||
- CRAWLER_BOOTNODES=testnet_sprs
|
- CRAWLER_BOOTNODES=testnet_sprs
|
||||||
- CRAWLER_STEPDELAY=3000
|
- CRAWLER_STEPDELAY=3000
|
||||||
- CRAWLER_REVISITDELAY=1440
|
- CRAWLER_REVISITDELAY=1440
|
||||||
ports:
|
ports:
|
||||||
- 8008:8008/tcp # Metrics
|
- 8008:8008/tcp # Metrics
|
||||||
- 8090:8090/udp # DHT discovery
|
- 8090:8090/udp # DHT discovery
|
||||||
volumes:
|
volumes:
|
||||||
- ./crawler_data:/crawler_data:z
|
- ./crawler_data:/crawler_data:z
|
||||||
|
|||||||
14
env.sh
14
env.sh
@ -1,7 +1,7 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
|
# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
|
||||||
# and we fall back to a Zsh-specific special var to also support Zsh.
|
# and we fall back to a Zsh-specific special var to also support Zsh.
|
||||||
REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
|
REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
|
||||||
ABS_PATH="$(cd ${REL_PATH}; pwd)"
|
ABS_PATH="$(cd ${REL_PATH}; pwd)"
|
||||||
source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
|
source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user