mirror of
https://github.com/logos-storage/logos-storage-network-crawler.git
synced 2026-01-02 05:23:07 +00:00
adjusting line endings
This commit is contained in:
thatben
parent
59b4b25667
commit
d71e1f1906
@ -1,6 +1,6 @@
|
||||
.github
|
||||
build
|
||||
docs
|
||||
metrics
|
||||
nimcache
|
||||
tests
|
||||
.github
|
||||
build
|
||||
docs
|
||||
metrics
|
||||
nimcache
|
||||
tests
|
||||
|
||||
2
.gitattributes
vendored
Normal file
2
.gitattributes
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
env.sh text eol=lf
|
||||
docker/docker-entrypoint.sh text eol=lf
|
||||
42
.github/workflows/docker.yml
vendored
42
.github/workflows/docker.yml
vendored
@ -1,21 +1,21 @@
|
||||
name: Docker
|
||||
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||
with:
|
||||
docker_file: docker/crawler.Dockerfile
|
||||
dockerhub_repo: codexstorage/codex-network-crawler
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
name: Docker
|
||||
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
|
||||
with:
|
||||
docker_file: docker/crawler.Dockerfile
|
||||
dockerhub_repo: codexstorage/codex-network-crawler
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
|
||||
86
.github/workflows/main.yml
vendored
86
.github/workflows/main.yml
vendored
@ -1,43 +1,43 @@
|
||||
name: CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
linting:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Check `nph` formatting
|
||||
uses: arnetheduck/nph-action@v1
|
||||
with:
|
||||
version: 0.6.1
|
||||
options: "./"
|
||||
fail: true
|
||||
suggest: true
|
||||
|
||||
test:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macOS-latest, windows-latest]
|
||||
nim: [2.2.4]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Build Nim and dependencies
|
||||
run: |
|
||||
which gcc
|
||||
gcc --version
|
||||
make update
|
||||
./env.sh nim --version
|
||||
make
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
make test
|
||||
|
||||
- name: Build crawler
|
||||
run: |
|
||||
make
|
||||
name: CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
linting:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Check `nph` formatting
|
||||
uses: arnetheduck/nph-action@v1
|
||||
with:
|
||||
version: 0.6.1
|
||||
options: "./"
|
||||
fail: true
|
||||
suggest: true
|
||||
|
||||
test:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, macOS-latest, windows-latest]
|
||||
nim: [2.2.4]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Build Nim and dependencies
|
||||
run: |
|
||||
which gcc
|
||||
gcc --version
|
||||
make update
|
||||
./env.sh nim --version
|
||||
make
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
make test
|
||||
|
||||
- name: Build crawler
|
||||
run: |
|
||||
make
|
||||
|
||||
16
.gitignore
vendored
16
.gitignore
vendored
@ -1,9 +1,9 @@
|
||||
*
|
||||
!*.*
|
||||
!*/
|
||||
nimble.paths
|
||||
nimbus-build-system.paths
|
||||
vendor/*
|
||||
*.exe
|
||||
crawler_data
|
||||
*
|
||||
!*.*
|
||||
!*/
|
||||
nimble.paths
|
||||
nimbus-build-system.paths
|
||||
vendor/*
|
||||
*.exe
|
||||
crawler_data
|
||||
.update.timestamp
|
||||
|
||||
394
Makefile
394
Makefile
@ -1,197 +1,197 @@
|
||||
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
|
||||
# either of:
|
||||
# - Apache License, version 2.0
|
||||
# - MIT license
|
||||
# at your option. This file may not be copied, modified, or distributed except
|
||||
# according to those terms.
|
||||
|
||||
# This is the Nim version used locally and in regular CI builds.
|
||||
# Can be a specific version tag, a branch name, or a commit hash.
|
||||
# Can be overridden by setting the NIM_COMMIT environment variable
|
||||
# before calling make.
|
||||
#
|
||||
# For readability in CI, if NIM_COMMIT is set to "pinned",
|
||||
# this will also default to the version pinned here.
|
||||
#
|
||||
# If NIM_COMMIT is set to "nimbusbuild", this will use the
|
||||
# version pinned by nimbus-build-system.
|
||||
PINNED_NIM_VERSION := v2.2.4
|
||||
|
||||
ifeq ($(NIM_COMMIT),)
|
||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||
else ifeq ($(NIM_COMMIT),pinned)
|
||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||
endif
|
||||
|
||||
ifeq ($(NIM_COMMIT),nimbusbuild)
|
||||
undefine NIM_COMMIT
|
||||
else
|
||||
export NIM_COMMIT
|
||||
endif
|
||||
|
||||
SHELL := bash # the shell used internally by Make
|
||||
|
||||
# used inside the included makefiles
|
||||
BUILD_SYSTEM_DIR := vendor/nimbus-build-system
|
||||
|
||||
# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
|
||||
# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
|
||||
DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
|
||||
|
||||
LINK_PCRE := 0
|
||||
|
||||
ifeq ($(OS),Windows_NT)
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
|
||||
ARCH = x86_64
|
||||
endif
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
|
||||
ARCH = arm64
|
||||
endif
|
||||
else
|
||||
UNAME_P := $(shell uname -m)
|
||||
ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
|
||||
ARCH = x86_64
|
||||
endif
|
||||
ifneq ($(filter $(UNAME_P), aarch64 arm),)
|
||||
ARCH = arm64
|
||||
endif
|
||||
endif
|
||||
|
||||
ifeq ($(ARCH), x86_64)
|
||||
CXXFLAGS ?= -std=c++17 -mssse3
|
||||
else
|
||||
CXXFLAGS ?= -std=c++17
|
||||
endif
|
||||
export CXXFLAGS
|
||||
|
||||
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
|
||||
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
|
||||
|
||||
.PHONY: \
|
||||
all \
|
||||
clean \
|
||||
coverage \
|
||||
deps \
|
||||
libbacktrace \
|
||||
test \
|
||||
update
|
||||
|
||||
ifeq ($(NIM_PARAMS),)
|
||||
# "variables.mk" was not included, so we update the submodules.
|
||||
GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
|
||||
.DEFAULT:
|
||||
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
|
||||
$(GIT_SUBMODULE_UPDATE); \
|
||||
echo
|
||||
# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
|
||||
# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
|
||||
#
|
||||
# After restarting, it will execute its original goal, so we don't have to start a child Make here
|
||||
# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
|
||||
|
||||
else # "variables.mk" was included. Business as usual until the end of this file.
|
||||
|
||||
# default target, because it's the first one that doesn't start with '.'
|
||||
|
||||
# Builds the crawler binary
|
||||
all: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims
|
||||
|
||||
# must be included after the default target
|
||||
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
|
||||
|
||||
# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
|
||||
ifeq ($(USE_LIBBACKTRACE), 0)
|
||||
NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
|
||||
else
|
||||
NIM_PARAMS := $(NIM_PARAMS) -d:release
|
||||
endif
|
||||
|
||||
deps: | deps-common nat-libs
|
||||
ifneq ($(USE_LIBBACKTRACE), 0)
|
||||
deps: | libbacktrace
|
||||
endif
|
||||
|
||||
update: | update-common
|
||||
|
||||
# detecting the os
|
||||
ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
|
||||
detected_OS := Windows
|
||||
else ifeq ($(strip $(shell uname)),Darwin)
|
||||
detected_OS := macOS
|
||||
else
|
||||
# e.g. Linux
|
||||
detected_OS := $(strip $(shell uname))
|
||||
endif
|
||||
|
||||
# Builds and run a part of the test suite
|
||||
test: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
|
||||
|
||||
# nim-libbacktrace
|
||||
LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
|
||||
libbacktrace:
|
||||
ifeq ($(detected_OS), Windows)
|
||||
# MSYS2 detection
|
||||
ifneq ($(MSYSTEM),)
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'"
|
||||
else
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
||||
endif
|
||||
else
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
||||
endif
|
||||
|
||||
# usual cleaning
|
||||
clean: | clean-common
|
||||
rm -rf build
|
||||
ifneq ($(USE_LIBBACKTRACE), 0)
|
||||
+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
|
||||
endif
|
||||
|
||||
############
|
||||
## Format ##
|
||||
############
|
||||
.PHONY: build-nph install-nph-hook clean-nph print-nph-path
|
||||
|
||||
# Default location for nph binary shall be next to nim binary to make it available on the path.
|
||||
NPH:=$(shell dirname $(NIM_BINARY))/nph
|
||||
|
||||
build-nph:
|
||||
ifeq ("$(wildcard $(NPH))","")
|
||||
$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
|
||||
mv vendor/nph/src/nph $(shell dirname $(NPH))
|
||||
echo "nph utility is available at " $(NPH)
|
||||
endif
|
||||
|
||||
GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
|
||||
|
||||
install-nph-hook: build-nph
|
||||
ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
|
||||
cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
|
||||
else
|
||||
echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
|
||||
exit 1
|
||||
endif
|
||||
|
||||
nph/%: build-nph
|
||||
echo -e $(FORMAT_MSG) "nph/$*" && \
|
||||
$(NPH) $*
|
||||
|
||||
format:
|
||||
$(NPH) *.nim
|
||||
$(NPH) codexcrawler/
|
||||
$(NPH) tests/
|
||||
|
||||
clean-nph:
|
||||
rm -f $(NPH)
|
||||
|
||||
# To avoid hardcoding nph binary location in several places
|
||||
print-nph-path:
|
||||
echo "$(NPH)"
|
||||
|
||||
clean: | clean-nph
|
||||
|
||||
endif # "variables.mk" was not included
|
||||
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
|
||||
# either of:
|
||||
# - Apache License, version 2.0
|
||||
# - MIT license
|
||||
# at your option. This file may not be copied, modified, or distributed except
|
||||
# according to those terms.
|
||||
|
||||
# This is the Nim version used locally and in regular CI builds.
|
||||
# Can be a specific version tag, a branch name, or a commit hash.
|
||||
# Can be overridden by setting the NIM_COMMIT environment variable
|
||||
# before calling make.
|
||||
#
|
||||
# For readability in CI, if NIM_COMMIT is set to "pinned",
|
||||
# this will also default to the version pinned here.
|
||||
#
|
||||
# If NIM_COMMIT is set to "nimbusbuild", this will use the
|
||||
# version pinned by nimbus-build-system.
|
||||
PINNED_NIM_VERSION := v2.2.4
|
||||
|
||||
ifeq ($(NIM_COMMIT),)
|
||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||
else ifeq ($(NIM_COMMIT),pinned)
|
||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||
endif
|
||||
|
||||
ifeq ($(NIM_COMMIT),nimbusbuild)
|
||||
undefine NIM_COMMIT
|
||||
else
|
||||
export NIM_COMMIT
|
||||
endif
|
||||
|
||||
SHELL := bash # the shell used internally by Make
|
||||
|
||||
# used inside the included makefiles
|
||||
BUILD_SYSTEM_DIR := vendor/nimbus-build-system
|
||||
|
||||
# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
|
||||
# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
|
||||
DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
|
||||
|
||||
LINK_PCRE := 0
|
||||
|
||||
ifeq ($(OS),Windows_NT)
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
|
||||
ARCH = x86_64
|
||||
endif
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
|
||||
ARCH = arm64
|
||||
endif
|
||||
else
|
||||
UNAME_P := $(shell uname -m)
|
||||
ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
|
||||
ARCH = x86_64
|
||||
endif
|
||||
ifneq ($(filter $(UNAME_P), aarch64 arm),)
|
||||
ARCH = arm64
|
||||
endif
|
||||
endif
|
||||
|
||||
ifeq ($(ARCH), x86_64)
|
||||
CXXFLAGS ?= -std=c++17 -mssse3
|
||||
else
|
||||
CXXFLAGS ?= -std=c++17
|
||||
endif
|
||||
export CXXFLAGS
|
||||
|
||||
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
|
||||
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
|
||||
|
||||
.PHONY: \
|
||||
all \
|
||||
clean \
|
||||
coverage \
|
||||
deps \
|
||||
libbacktrace \
|
||||
test \
|
||||
update
|
||||
|
||||
ifeq ($(NIM_PARAMS),)
|
||||
# "variables.mk" was not included, so we update the submodules.
|
||||
GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
|
||||
.DEFAULT:
|
||||
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
|
||||
$(GIT_SUBMODULE_UPDATE); \
|
||||
echo
|
||||
# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
|
||||
# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
|
||||
#
|
||||
# After restarting, it will execute its original goal, so we don't have to start a child Make here
|
||||
# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
|
||||
|
||||
else # "variables.mk" was included. Business as usual until the end of this file.
|
||||
|
||||
# default target, because it's the first one that doesn't start with '.'
|
||||
|
||||
# Builds the crawler binary
|
||||
all: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims
|
||||
|
||||
# must be included after the default target
|
||||
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
|
||||
|
||||
# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
|
||||
ifeq ($(USE_LIBBACKTRACE), 0)
|
||||
NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
|
||||
else
|
||||
NIM_PARAMS := $(NIM_PARAMS) -d:release
|
||||
endif
|
||||
|
||||
deps: | deps-common nat-libs
|
||||
ifneq ($(USE_LIBBACKTRACE), 0)
|
||||
deps: | libbacktrace
|
||||
endif
|
||||
|
||||
update: | update-common
|
||||
|
||||
# detecting the os
|
||||
ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
|
||||
detected_OS := Windows
|
||||
else ifeq ($(strip $(shell uname)),Darwin)
|
||||
detected_OS := macOS
|
||||
else
|
||||
# e.g. Linux
|
||||
detected_OS := $(strip $(shell uname))
|
||||
endif
|
||||
|
||||
# Builds and run a part of the test suite
|
||||
test: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
|
||||
|
||||
# nim-libbacktrace
|
||||
LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
|
||||
libbacktrace:
|
||||
ifeq ($(detected_OS), Windows)
|
||||
# MSYS2 detection
|
||||
ifneq ($(MSYSTEM),)
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'"
|
||||
else
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
||||
endif
|
||||
else
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
||||
endif
|
||||
|
||||
# usual cleaning
|
||||
clean: | clean-common
|
||||
rm -rf build
|
||||
ifneq ($(USE_LIBBACKTRACE), 0)
|
||||
+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
|
||||
endif
|
||||
|
||||
############
|
||||
## Format ##
|
||||
############
|
||||
.PHONY: build-nph install-nph-hook clean-nph print-nph-path
|
||||
|
||||
# Default location for nph binary shall be next to nim binary to make it available on the path.
|
||||
NPH:=$(shell dirname $(NIM_BINARY))/nph
|
||||
|
||||
build-nph:
|
||||
ifeq ("$(wildcard $(NPH))","")
|
||||
$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
|
||||
mv vendor/nph/src/nph $(shell dirname $(NPH))
|
||||
echo "nph utility is available at " $(NPH)
|
||||
endif
|
||||
|
||||
GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
|
||||
|
||||
install-nph-hook: build-nph
|
||||
ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
|
||||
cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
|
||||
else
|
||||
echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
|
||||
exit 1
|
||||
endif
|
||||
|
||||
nph/%: build-nph
|
||||
echo -e $(FORMAT_MSG) "nph/$*" && \
|
||||
$(NPH) $*
|
||||
|
||||
format:
|
||||
$(NPH) *.nim
|
||||
$(NPH) codexcrawler/
|
||||
$(NPH) tests/
|
||||
|
||||
clean-nph:
|
||||
rm -f $(NPH)
|
||||
|
||||
# To avoid hardcoding nph binary location in several places
|
||||
print-nph-path:
|
||||
echo "$(NPH)"
|
||||
|
||||
clean: | clean-nph
|
||||
|
||||
endif # "variables.mk" was not included
|
||||
|
||||
54
README.md
54
README.md
@ -1,27 +1,27 @@
|
||||
# Codex Network Crawler
|
||||
|
||||
|
||||
|
||||
[](https://opensource.org/licenses/Apache-2.0)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](#stability)
|
||||
[](https://github.com/codex-storage/nim-codex-dht/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amaster)
|
||||
[](https://codecov.io/gh/codex-storage/nim-codex-dht)
|
||||
|
||||
# !! Work in Progress !!
|
||||
|
||||
This project uses nim-codex-dht, nim-libp2p, nim-ethers, and nim-metrics to create a metrics service. The crawler will traverse the Codex network and produce metrics such as:
|
||||
- Number of DHT nodes (alive vs total)
|
||||
- P2P connectivity (percentage)
|
||||
- Storage contract statistics (created, total size, average size, average duration, pricing information??)
|
||||
|
||||
Metrics are published from a scrape target.
|
||||
|
||||
# Usage
|
||||
|
||||
```sh
|
||||
nimble format
|
||||
nimble build
|
||||
nimble test
|
||||
nimble run
|
||||
```
|
||||
# Codex Network Crawler
|
||||
|
||||
|
||||
|
||||
[](https://opensource.org/licenses/Apache-2.0)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](#stability)
|
||||
[](https://github.com/codex-storage/nim-codex-dht/actions/workflows/ci.yml?query=workflow%3ACI+branch%3Amaster)
|
||||
[](https://codecov.io/gh/codex-storage/nim-codex-dht)
|
||||
|
||||
# !! Work in Progress !!
|
||||
|
||||
This project uses nim-codex-dht, nim-libp2p, nim-ethers, and nim-metrics to create a metrics service. The crawler will traverse the Codex network and produce metrics such as:
|
||||
- Number of DHT nodes (alive vs total)
|
||||
- P2P connectivity (percentage)
|
||||
- Storage contract statistics (created, total size, average size, average duration, pricing information??)
|
||||
|
||||
Metrics are published from a scrape target.
|
||||
|
||||
# Usage
|
||||
|
||||
```sh
|
||||
nimble format
|
||||
nimble build
|
||||
nimble test
|
||||
nimble run
|
||||
```
|
||||
|
||||
@ -1,39 +1,39 @@
|
||||
# Package
|
||||
|
||||
version = "0.0.1"
|
||||
author = "Codex core contributors"
|
||||
description = "Crawler for Codex networks"
|
||||
license = "MIT"
|
||||
skipDirs = @["tests"]
|
||||
bin = @["codexcrawler"]
|
||||
binDir = "build"
|
||||
|
||||
# Dependencies
|
||||
requires "nim >= 2.0.14 & < 3.0.0"
|
||||
requires "secp256k1 >= 0.6.0 & < 0.7.0"
|
||||
requires "protobuf_serialization >= 0.3.0 & < 0.4.0"
|
||||
requires "nimcrypto >= 0.6.2 & < 0.7.0"
|
||||
requires "bearssl >= 0.2.5 & < 0.3.0"
|
||||
requires "chronicles >= 0.10.2 & < 0.11.0"
|
||||
requires "chronos >= 4.0.3 & < 4.1.0"
|
||||
# requires "libp2p >= 1.5.0 & < 2.0.0"
|
||||
requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses.
|
||||
requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3"
|
||||
|
||||
requires "metrics >= 0.1.0 & < 0.2.0"
|
||||
requires "stew >= 0.2.0 & < 0.3.0"
|
||||
requires "stint >= 0.8.1 & < 0.9.0"
|
||||
requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0"
|
||||
requires "questionable >= 0.10.15 & < 0.11.0"
|
||||
requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59"
|
||||
requires "docopt >= 0.7.1 & < 1.0.0"
|
||||
requires "nph >= 0.6.1 & < 1.0.0"
|
||||
requires "ethers >= 1.0.0 & < 2.0.0"
|
||||
|
||||
task format, "Formatting...":
|
||||
exec "nph ./"
|
||||
|
||||
task test, "Run tests":
|
||||
exec "nimble install -d -y"
|
||||
withDir "tests":
|
||||
exec "nimble test"
|
||||
# Package
|
||||
|
||||
version = "0.0.1"
|
||||
author = "Codex core contributors"
|
||||
description = "Crawler for Codex networks"
|
||||
license = "MIT"
|
||||
skipDirs = @["tests"]
|
||||
bin = @["codexcrawler"]
|
||||
binDir = "build"
|
||||
|
||||
# Dependencies
|
||||
requires "nim >= 2.0.14 & < 3.0.0"
|
||||
requires "secp256k1 >= 0.6.0 & < 0.7.0"
|
||||
requires "protobuf_serialization >= 0.3.0 & < 0.4.0"
|
||||
requires "nimcrypto >= 0.6.2 & < 0.7.0"
|
||||
requires "bearssl >= 0.2.5 & < 0.3.0"
|
||||
requires "chronicles >= 0.10.2 & < 0.11.0"
|
||||
requires "chronos >= 4.0.3 & < 4.1.0"
|
||||
# requires "libp2p >= 1.5.0 & < 2.0.0"
|
||||
requires "libp2p#036e110a6080fba1a1662c58cfd8c21f9a548021" # Same as codex-0.2.0 uses.
|
||||
requires "https://github.com/codex-storage/nim-poseidon2#4e2c6e619b2f2859aaa4b2aed2f346ea4d0c67a3"
|
||||
|
||||
requires "metrics >= 0.1.0 & < 0.2.0"
|
||||
requires "stew >= 0.2.0 & < 0.3.0"
|
||||
requires "stint >= 0.8.1 & < 0.9.0"
|
||||
requires "https://github.com/codex-storage/nim-datastore >= 0.2.0 & < 0.3.0"
|
||||
requires "questionable >= 0.10.15 & < 0.11.0"
|
||||
requires "https://github.com/codex-storage/nim-codex-dht#89b281fc8d84097d4cd1f8a53fac68bd23433f59"
|
||||
requires "docopt >= 0.7.1 & < 1.0.0"
|
||||
requires "nph >= 0.6.1 & < 1.0.0"
|
||||
requires "ethers >= 1.0.0 & < 2.0.0"
|
||||
|
||||
task format, "Formatting...":
|
||||
exec "nph ./"
|
||||
|
||||
task test, "Run tests":
|
||||
exec "nimble install -d -y"
|
||||
withDir "tests":
|
||||
exec "nimble test"
|
||||
|
||||
@ -6,31 +6,31 @@ import pkg/codexdht
|
||||
import ./utils/version
|
||||
|
||||
let doc =
|
||||
"""
|
||||
Codex Network Crawler. Generates network metrics.
|
||||
|
||||
Usage:
|
||||
codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>] [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>] [--requestCheckDelay=<m>]
|
||||
|
||||
Options:
|
||||
--logLevel=<l> Sets log level [default: INFO]
|
||||
--publicIp=<a> Public IP address where this instance is reachable.
|
||||
--metricsAddress=<ip> Listen address of the metrics server [default: 0.0.0.0]
|
||||
--metricsPort=<p> Listen HTTP port of the metrics server [default: 8008]
|
||||
--dataDir=<dir> Directory for storing data [default: crawler_data]
|
||||
--discoveryPort=<p> Port used for DHT [default: 8090]
|
||||
--bootNodes=<n> Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
|
||||
|
||||
--dhtEnable=<e> Set to "1" to enable DHT crawler [default: 1]
|
||||
--stepDelay=<ms> Delay in milliseconds per node visit [default: 1000]
|
||||
--revisitDelay=<m> Delay in minutes after which a node can be revisited [default: 60]
|
||||
--checkDelay=<m> Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
|
||||
--expiryDelay=<m> Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
|
||||
|
||||
--marketplaceEnable=<e> Set to "1" to enable marketplace metrics [default: 1]
|
||||
--ethProvider=<a> Address including http(s) or ws of the eth provider
|
||||
--marketplaceAddress=<a> Eth address of Codex contracts deployment
|
||||
--requestCheckDelay=<m> Delay in minutes after which storage contract status is (re)checked [default: 10]
|
||||
"""
|
||||
Codex Network Crawler. Generates network metrics.
|
||||
|
||||
Usage:
|
||||
codexcrawler [--logLevel=<l>] [--publicIp=<a>] [--metricsAddress=<ip>] [--metricsPort=<p>] [--dataDir=<dir>] [--discoveryPort=<p>] [--bootNodes=<n>] [--dhtEnable=<e>] [--stepDelay=<ms>] [--revisitDelay=<m>] [--checkDelay=<m>] [--expiryDelay=<m>] [--marketplaceEnable=<e>] [--ethProvider=<a>] [--marketplaceAddress=<a>] [--requestCheckDelay=<m>]
|
||||
|
||||
Options:
|
||||
--logLevel=<l> Sets log level [default: INFO]
|
||||
--publicIp=<a> Public IP address where this instance is reachable.
|
||||
--metricsAddress=<ip> Listen address of the metrics server [default: 0.0.0.0]
|
||||
--metricsPort=<p> Listen HTTP port of the metrics server [default: 8008]
|
||||
--dataDir=<dir> Directory for storing data [default: crawler_data]
|
||||
--discoveryPort=<p> Port used for DHT [default: 8090]
|
||||
--bootNodes=<n> Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
|
||||
|
||||
--dhtEnable=<e> Set to "1" to enable DHT crawler [default: 1]
|
||||
--stepDelay=<ms> Delay in milliseconds per node visit [default: 1000]
|
||||
--revisitDelay=<m> Delay in minutes after which a node can be revisited [default: 60]
|
||||
--checkDelay=<m> Delay with which the 'revisitDelay' is checked for all known nodes [default: 10]
|
||||
--expiryDelay=<m> Delay in minutes after which unresponsive nodes are discarded [default: 1440] (24h)
|
||||
|
||||
--marketplaceEnable=<e> Set to "1" to enable marketplace metrics [default: 1]
|
||||
--ethProvider=<a> Address including http(s) or ws of the eth provider
|
||||
--marketplaceAddress=<a> Eth address of Codex contracts deployment
|
||||
--requestCheckDelay=<m> Delay in minutes after which storage contract status is (re)checked [default: 10]
|
||||
"""
|
||||
|
||||
import strutils
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
These are copied from nim-codex v0.2.0.
|
||||
There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled.
|
||||
These are copied from nim-codex v0.2.0.
|
||||
There are plans to extract/refactor the contract interoperability code from nim-codex into its own submodule. But this isn't prioritized atm. So we're copying it here until that's been handled.
|
||||
|
||||
@ -1,36 +1,36 @@
|
||||
# Variables
|
||||
ARG BUILDER=ubuntu:24.04
|
||||
ARG IMAGE=${BUILDER}
|
||||
ARG BUILD_HOME=/src
|
||||
ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
|
||||
ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}"
|
||||
ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1}
|
||||
ARG APP_HOME=/crawler
|
||||
|
||||
# Build
|
||||
FROM ${BUILDER} AS builder
|
||||
ARG BUILD_HOME
|
||||
ARG MAKE_PARALLEL
|
||||
ARG NIMFLAGS
|
||||
ARG USE_LIBBACKTRACE
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y git cmake curl make bash build-essential
|
||||
|
||||
WORKDIR ${BUILD_HOME}
|
||||
COPY . .
|
||||
RUN make -j ${MAKE_PARALLEL} update
|
||||
RUN make -j ${MAKE_PARALLEL}
|
||||
|
||||
# Create
|
||||
FROM ${IMAGE}
|
||||
ARG BUILD_HOME
|
||||
ARG APP_HOME
|
||||
|
||||
WORKDIR ${APP_HOME}
|
||||
COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
|
||||
COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
|
||||
RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
CMD ["codexcrawler"]
|
||||
# Variables
|
||||
ARG BUILDER=ubuntu:24.04
|
||||
ARG IMAGE=${BUILDER}
|
||||
ARG BUILD_HOME=/src
|
||||
ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
|
||||
ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}"
|
||||
ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1}
|
||||
ARG APP_HOME=/crawler
|
||||
|
||||
# Build
|
||||
FROM ${BUILDER} AS builder
|
||||
ARG BUILD_HOME
|
||||
ARG MAKE_PARALLEL
|
||||
ARG NIMFLAGS
|
||||
ARG USE_LIBBACKTRACE
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y git cmake curl make bash build-essential
|
||||
|
||||
WORKDIR ${BUILD_HOME}
|
||||
COPY . .
|
||||
RUN make -j ${MAKE_PARALLEL} update
|
||||
RUN make -j ${MAKE_PARALLEL}
|
||||
|
||||
# Create
|
||||
FROM ${IMAGE}
|
||||
ARG BUILD_HOME
|
||||
ARG APP_HOME
|
||||
|
||||
WORKDIR ${APP_HOME}
|
||||
COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
|
||||
COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
|
||||
RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
CMD ["codexcrawler"]
|
||||
|
||||
@ -1,20 +1,20 @@
|
||||
services:
|
||||
crawler1:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: ./docker/crawler.Dockerfile
|
||||
environment:
|
||||
- CRAWLER_LOGLEVEL=TRACE
|
||||
# - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage
|
||||
- CRAWLER_METRICSADDRESS=0.0.0.0
|
||||
- CRAWLER_METRICSPORT=8008
|
||||
- CRAWLER_DATADIR=crawler_data
|
||||
- CRAWLER_DISCPORT=8090
|
||||
- CRAWLER_BOOTNODES=testnet_sprs
|
||||
- CRAWLER_STEPDELAY=3000
|
||||
- CRAWLER_REVISITDELAY=1440
|
||||
ports:
|
||||
- 8008:8008/tcp # Metrics
|
||||
- 8090:8090/udp # DHT discovery
|
||||
volumes:
|
||||
- ./crawler_data:/crawler_data:z
|
||||
services:
|
||||
crawler1:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: ./docker/crawler.Dockerfile
|
||||
environment:
|
||||
- CRAWLER_LOGLEVEL=TRACE
|
||||
# - CRAWLER_PUBLICIP= Set to override CURL to ip.codex.storage
|
||||
- CRAWLER_METRICSADDRESS=0.0.0.0
|
||||
- CRAWLER_METRICSPORT=8008
|
||||
- CRAWLER_DATADIR=crawler_data
|
||||
- CRAWLER_DISCPORT=8090
|
||||
- CRAWLER_BOOTNODES=testnet_sprs
|
||||
- CRAWLER_STEPDELAY=3000
|
||||
- CRAWLER_REVISITDELAY=1440
|
||||
ports:
|
||||
- 8008:8008/tcp # Metrics
|
||||
- 8090:8090/udp # DHT discovery
|
||||
volumes:
|
||||
- ./crawler_data:/crawler_data:z
|
||||
|
||||
14
env.sh
14
env.sh
@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
|
||||
# and we fall back to a Zsh-specific special var to also support Zsh.
|
||||
REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
|
||||
ABS_PATH="$(cd ${REL_PATH}; pwd)"
|
||||
source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
|
||||
# and we fall back to a Zsh-specific special var to also support Zsh.
|
||||
REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
|
||||
ABS_PATH="$(cd ${REL_PATH}; pwd)"
|
||||
source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user