mirror of
https://github.com/logos-storage/logos-storage-network-crawler.git
synced 2026-02-25 07:53:07 +00:00
wip
This commit is contained in:
thatben
parent
cc02dbcbfc
commit
83aa7839d2
10
.gitignore
vendored
10
.gitignore
vendored
@ -1,17 +1,9 @@
|
||||
*
|
||||
!*.*
|
||||
!*/
|
||||
coverage
|
||||
nimcache
|
||||
tests/testAll
|
||||
nimble.develop
|
||||
nimble.paths
|
||||
nim.cfg
|
||||
nimbus-build-system.paths
|
||||
vendor/*
|
||||
NimBinaries
|
||||
.update.timestamp
|
||||
*.dSYM
|
||||
.vscode/*
|
||||
*.exe
|
||||
crawler_data
|
||||
.update.timestamp
|
||||
|
||||
7
.gitmodules
vendored
7
.gitmodules
vendored
@ -16,8 +16,8 @@
|
||||
[submodule "vendor/nim-secp256k1"]
|
||||
path = vendor/nim-secp256k1
|
||||
url = https://github.com/status-im/nim-secp256k1.git
|
||||
[submodule "nim-stew"]
|
||||
path = nim-stew
|
||||
[submodule "vendor/nim-stew"]
|
||||
path = vendor/nim-stew
|
||||
url = https://github.com/status-im/nim-stew.git
|
||||
[submodule "vendor/questionable"]
|
||||
path = vendor/questionable
|
||||
@ -97,3 +97,6 @@
|
||||
[submodule "vendor/nph"]
|
||||
path = vendor/nph
|
||||
url = https://github.com/arnetheduck/nph.git
|
||||
[submodule "vendor/nim-faststreams"]
|
||||
path = vendor/nim-faststreams
|
||||
url = https://github.com/status-im/nim-faststreams.git
|
||||
|
||||
222
Makefile
Normal file
222
Makefile
Normal file
@ -0,0 +1,222 @@
|
||||
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
|
||||
# either of:
|
||||
# - Apache License, version 2.0
|
||||
# - MIT license
|
||||
# at your option. This file may not be copied, modified, or distributed except
|
||||
# according to those terms.
|
||||
|
||||
# This is the Nim version used locally and in regular CI builds.
|
||||
# Can be a specific version tag, a branch name, or a commit hash.
|
||||
# Can be overridden by setting the NIM_COMMIT environment variable
|
||||
# before calling make.
|
||||
#
|
||||
# For readability in CI, if NIM_COMMIT is set to "pinned",
|
||||
# this will also default to the version pinned here.
|
||||
#
|
||||
# If NIM_COMMIT is set to "nimbusbuild", this will use the
|
||||
# version pinned by nimbus-build-system.
|
||||
PINNED_NIM_VERSION := v2.2.4
|
||||
|
||||
ifeq ($(NIM_COMMIT),)
|
||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||
else ifeq ($(NIM_COMMIT),pinned)
|
||||
NIM_COMMIT := $(PINNED_NIM_VERSION)
|
||||
endif
|
||||
|
||||
ifeq ($(NIM_COMMIT),nimbusbuild)
|
||||
undefine NIM_COMMIT
|
||||
else
|
||||
export NIM_COMMIT
|
||||
endif
|
||||
|
||||
SHELL := bash # the shell used internally by Make
|
||||
|
||||
# used inside the included makefiles
|
||||
BUILD_SYSTEM_DIR := vendor/nimbus-build-system
|
||||
|
||||
# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
|
||||
# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
|
||||
DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
|
||||
|
||||
LINK_PCRE := 0
|
||||
|
||||
ifeq ($(OS),Windows_NT)
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), AMD64)
|
||||
ARCH = x86_64
|
||||
endif
|
||||
ifeq ($(PROCESSOR_ARCHITECTURE), ARM64)
|
||||
ARCH = arm64
|
||||
endif
|
||||
else
|
||||
UNAME_P := $(shell uname -m)
|
||||
ifneq ($(filter $(UNAME_P), i686 i386 x86_64),)
|
||||
ARCH = x86_64
|
||||
endif
|
||||
ifneq ($(filter $(UNAME_P), aarch64 arm),)
|
||||
ARCH = arm64
|
||||
endif
|
||||
endif
|
||||
|
||||
ifeq ($(ARCH), x86_64)
|
||||
CXXFLAGS ?= -std=c++17 -mssse3
|
||||
else
|
||||
CXXFLAGS ?= -std=c++17
|
||||
endif
|
||||
export CXXFLAGS
|
||||
|
||||
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
|
||||
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
|
||||
|
||||
.PHONY: \
|
||||
all \
|
||||
clean \
|
||||
coverage \
|
||||
deps \
|
||||
libbacktrace \
|
||||
test \
|
||||
update
|
||||
|
||||
ifeq ($(NIM_PARAMS),)
|
||||
# "variables.mk" was not included, so we update the submodules.
|
||||
GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
|
||||
.DEFAULT:
|
||||
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
|
||||
$(GIT_SUBMODULE_UPDATE); \
|
||||
echo
|
||||
# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
|
||||
# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
|
||||
#
|
||||
# After restarting, it will execute its original goal, so we don't have to start a child Make here
|
||||
# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
|
||||
|
||||
else # "variables.mk" was included. Business as usual until the end of this file.
|
||||
|
||||
# default target, because it's the first one that doesn't start with '.'
|
||||
|
||||
# Builds the crawler binary
|
||||
all: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims
|
||||
|
||||
# must be included after the default target
|
||||
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
|
||||
|
||||
# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims
|
||||
ifeq ($(USE_LIBBACKTRACE), 0)
|
||||
NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace
|
||||
else
|
||||
NIM_PARAMS := $(NIM_PARAMS) -d:release
|
||||
endif
|
||||
|
||||
deps: | deps-common nat-libs
|
||||
ifneq ($(USE_LIBBACKTRACE), 0)
|
||||
deps: | libbacktrace
|
||||
endif
|
||||
|
||||
update: | update-common
|
||||
|
||||
# detecting the os
|
||||
ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10...
|
||||
detected_OS := Windows
|
||||
else ifeq ($(strip $(shell uname)),Darwin)
|
||||
detected_OS := macOS
|
||||
else
|
||||
# e.g. Linux
|
||||
detected_OS := $(strip $(shell uname))
|
||||
endif
|
||||
|
||||
# Builds and run a part of the test suite
|
||||
test: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims
|
||||
|
||||
# Builds and runs the smart contract tests
|
||||
testContracts: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
|
||||
|
||||
# Builds and runs the integration tests
|
||||
testIntegration: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims
|
||||
|
||||
# Builds and runs all tests (except for Taiko L2 tests)
|
||||
testAll: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims
|
||||
|
||||
# Builds and runs Taiko L2 tests
|
||||
testTaiko: | build deps
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims
|
||||
|
||||
# Builds and runs tool tests
|
||||
testTools: | cirdl
|
||||
echo -e $(BUILD_MSG) "build/$@" && \
|
||||
$(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims
|
||||
|
||||
# nim-libbacktrace
|
||||
LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0
|
||||
libbacktrace:
|
||||
ifeq ($(detected_OS), Windows)
|
||||
# MSYS2 detection
|
||||
ifneq ($(MSYSTEM),)
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'"
|
||||
else
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
||||
endif
|
||||
else
|
||||
+ $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS)
|
||||
endif
|
||||
|
||||
# usual cleaning
|
||||
clean: | clean-common
|
||||
rm -rf build
|
||||
ifneq ($(USE_LIBBACKTRACE), 0)
|
||||
+ $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT)
|
||||
endif
|
||||
|
||||
############
|
||||
## Format ##
|
||||
############
|
||||
.PHONY: build-nph install-nph-hook clean-nph print-nph-path
|
||||
|
||||
# Default location for nph binary shall be next to nim binary to make it available on the path.
|
||||
NPH:=$(shell dirname $(NIM_BINARY))/nph
|
||||
|
||||
build-nph:
|
||||
ifeq ("$(wildcard $(NPH))","")
|
||||
$(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \
|
||||
mv vendor/nph/src/nph $(shell dirname $(NPH))
|
||||
echo "nph utility is available at " $(NPH)
|
||||
endif
|
||||
|
||||
GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit
|
||||
|
||||
install-nph-hook: build-nph
|
||||
ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","")
|
||||
cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK)
|
||||
else
|
||||
echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override"
|
||||
exit 1
|
||||
endif
|
||||
|
||||
nph/%: build-nph
|
||||
echo -e $(FORMAT_MSG) "nph/$*" && \
|
||||
$(NPH) $*
|
||||
|
||||
format:
|
||||
$(NPH) *.nim
|
||||
$(NPH) codexcrawler/
|
||||
$(NPH) tests/
|
||||
|
||||
clean-nph:
|
||||
rm -f $(NPH)
|
||||
|
||||
# To avoid hardcoding nph binary location in several places
|
||||
print-nph-path:
|
||||
echo "$(NPH)"
|
||||
|
||||
clean: | clean-nph
|
||||
|
||||
endif # "variables.mk" was not included
|
||||
92
build.nims
92
build.nims
@ -29,95 +29,15 @@ proc test(name: string, srcDir = "tests/", params = "", lang = "c") =
|
||||
buildBinary name, srcDir, params
|
||||
exec "build/" & name
|
||||
|
||||
task codex, "build codex binary":
|
||||
buildBinary "codex",
|
||||
task codexcrawler, "build codexcrawler binary":
|
||||
buildBinary "codexcrawler",
|
||||
params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
|
||||
|
||||
task toolsCirdl, "build tools/cirdl binary":
|
||||
buildBinary "tools/cirdl/cirdl"
|
||||
|
||||
task testCodex, "Build & run Codex tests":
|
||||
test "testCodex", params = "-d:codex_enable_proof_failures=true"
|
||||
|
||||
task testContracts, "Build & run Codex Contract tests":
|
||||
test "testContracts"
|
||||
|
||||
task testIntegration, "Run integration tests":
|
||||
buildBinary "codex",
|
||||
params =
|
||||
"-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true"
|
||||
test "testIntegration"
|
||||
# use params to enable logging from the integration test executable
|
||||
# test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " &
|
||||
# "-d:chronicles_enabled_topics:integration:TRACE"
|
||||
task testCodexcrawler, "Build & run Codex Crawler tests":
|
||||
test "testCodexCrawler", params = "-d:codex_enable_proof_failures=true"
|
||||
|
||||
task build, "build codex binary":
|
||||
codexTask()
|
||||
codexCrawlerTask()
|
||||
|
||||
task test, "Run tests":
|
||||
testCodexTask()
|
||||
|
||||
task testTools, "Run Tools tests":
|
||||
toolsCirdlTask()
|
||||
test "testTools"
|
||||
|
||||
task testAll, "Run all tests (except for Taiko L2 tests)":
|
||||
testCodexTask()
|
||||
testContractsTask()
|
||||
testIntegrationTask()
|
||||
testToolsTask()
|
||||
|
||||
task testTaiko, "Run Taiko L2 tests":
|
||||
codexTask()
|
||||
test "testTaiko"
|
||||
|
||||
import strutils
|
||||
import os
|
||||
|
||||
task coverage, "generates code coverage report":
|
||||
var (output, exitCode) = gorgeEx("which lcov")
|
||||
if exitCode != 0:
|
||||
echo " ************************** ⛔️ ERROR ⛔️ **************************"
|
||||
echo " ** ERROR: lcov not found, it must be installed to run code **"
|
||||
echo " ** coverage locally **"
|
||||
echo " *****************************************************************"
|
||||
quit 1
|
||||
|
||||
(output, exitCode) = gorgeEx("gcov --version")
|
||||
if output.contains("Apple LLVM"):
|
||||
echo " ************************* ⚠️ WARNING ⚠️ *************************"
|
||||
echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **"
|
||||
echo " ** emulates an old version of gcov (4.2.0) and therefore **"
|
||||
echo " ** coverage results will differ than those on CI (which **"
|
||||
echo " ** uses a much newer version of gcov). **"
|
||||
echo " *****************************************************************"
|
||||
|
||||
var nimSrcs = " "
|
||||
for f in walkDirRec("codex", {pcFile}):
|
||||
if f.endswith(".nim"):
|
||||
nimSrcs.add " " & f.absolutePath.quoteShell()
|
||||
|
||||
echo "======== Running Tests ======== "
|
||||
test "coverage",
|
||||
srcDir = "tests/",
|
||||
params =
|
||||
" --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true"
|
||||
exec("rm nimcache/coverage/*.c")
|
||||
rmDir("coverage")
|
||||
mkDir("coverage")
|
||||
echo " ======== Running LCOV ======== "
|
||||
exec(
|
||||
"lcov --capture --keep-going --directory nimcache/coverage --output-file coverage/coverage.info"
|
||||
)
|
||||
exec(
|
||||
"lcov --extract coverage/coverage.info --keep-going --output-file coverage/coverage.f.info " &
|
||||
nimSrcs
|
||||
)
|
||||
echo " ======== Generating HTML coverage report ======== "
|
||||
exec("genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report ")
|
||||
echo " ======== Coverage report Done ======== "
|
||||
|
||||
task showCoverage, "open coverage html":
|
||||
echo " ======== Opening HTML coverage report in browser... ======== "
|
||||
if findExe("open") != "":
|
||||
exec("open coverage/report/index.html")
|
||||
testCodexCrawlerTask()
|
||||
|
||||
1
vendor/nim-faststreams
vendored
Submodule
1
vendor/nim-faststreams
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit c51315d0ae5eb2594d0bf41181d0e1aca1b3c01d
|
||||
0
nim-stew → vendor/nim-stew
vendored
0
nim-stew → vendor/nim-stew
vendored
Loading…
x
Reference in New Issue
Block a user