diff --git a/.gitignore b/.gitignore index ca02be0..fbe4999 100644 --- a/.gitignore +++ b/.gitignore @@ -1,17 +1,9 @@ * !*.* !*/ -coverage -nimcache -tests/testAll -nimble.develop nimble.paths -nim.cfg nimbus-build-system.paths vendor/* -NimBinaries -.update.timestamp -*.dSYM -.vscode/* *.exe crawler_data +.update.timestamp diff --git a/.gitmodules b/.gitmodules index 6b5e018..c058544 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,8 +16,8 @@ [submodule "vendor/nim-secp256k1"] path = vendor/nim-secp256k1 url = https://github.com/status-im/nim-secp256k1.git -[submodule "nim-stew"] - path = nim-stew +[submodule "vendor/nim-stew"] + path = vendor/nim-stew url = https://github.com/status-im/nim-stew.git [submodule "vendor/questionable"] path = vendor/questionable @@ -97,3 +97,6 @@ [submodule "vendor/nph"] path = vendor/nph url = https://github.com/arnetheduck/nph.git +[submodule "vendor/nim-faststreams"] + path = vendor/nim-faststreams + url = https://github.com/status-im/nim-faststreams.git diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d80a0d1 --- /dev/null +++ b/Makefile @@ -0,0 +1,222 @@ +# Copyright (c) 2020 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# This is the Nim version used locally and in regular CI builds. +# Can be a specific version tag, a branch name, or a commit hash. +# Can be overridden by setting the NIM_COMMIT environment variable +# before calling make. +# +# For readability in CI, if NIM_COMMIT is set to "pinned", +# this will also default to the version pinned here. +# +# If NIM_COMMIT is set to "nimbusbuild", this will use the +# version pinned by nimbus-build-system. +PINNED_NIM_VERSION := v2.2.4 + +ifeq ($(NIM_COMMIT),) +NIM_COMMIT := $(PINNED_NIM_VERSION) +else ifeq ($(NIM_COMMIT),pinned) +NIM_COMMIT := $(PINNED_NIM_VERSION) +endif + +ifeq ($(NIM_COMMIT),nimbusbuild) +undefine NIM_COMMIT +else +export NIM_COMMIT +endif + +SHELL := bash # the shell used internally by Make + +# used inside the included makefiles +BUILD_SYSTEM_DIR := vendor/nimbus-build-system + +# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics +# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker +DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure + +LINK_PCRE := 0 + +ifeq ($(OS),Windows_NT) + ifeq ($(PROCESSOR_ARCHITECTURE), AMD64) + ARCH = x86_64 + endif + ifeq ($(PROCESSOR_ARCHITECTURE), ARM64) + ARCH = arm64 + endif +else + UNAME_P := $(shell uname -m) + ifneq ($(filter $(UNAME_P), i686 i386 x86_64),) + ARCH = x86_64 + endif + ifneq ($(filter $(UNAME_P), aarch64 arm),) + ARCH = arm64 + endif +endif + +ifeq ($(ARCH), x86_64) + CXXFLAGS ?= -std=c++17 -mssse3 +else + CXXFLAGS ?= -std=c++17 +endif +export CXXFLAGS + +# we don't want an error here, so we can handle things later, in the ".DEFAULT" target +-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk + +.PHONY: \ + all \ + clean \ + coverage \ + deps \ + libbacktrace \ + test \ + update + +ifeq ($(NIM_PARAMS),) +# "variables.mk" was not included, so we update the submodules. +GIT_SUBMODULE_UPDATE := git submodule update --init --recursive +.DEFAULT: + +@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ + $(GIT_SUBMODULE_UPDATE); \ + echo +# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself: +# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles +# +# After restarting, it will execute its original goal, so we don't have to start a child Make here +# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great? + +else # "variables.mk" was included. Business as usual until the end of this file. + +# default target, because it's the first one that doesn't start with '.' + +# Builds the crawler binary +all: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim codexcrawler $(NIM_PARAMS) build.nims + +# must be included after the default target +-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk + +# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims +ifeq ($(USE_LIBBACKTRACE), 0) +NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace +else +NIM_PARAMS := $(NIM_PARAMS) -d:release +endif + +deps: | deps-common nat-libs +ifneq ($(USE_LIBBACKTRACE), 0) +deps: | libbacktrace +endif + +update: | update-common + +# detecting the os +ifeq ($(OS),Windows_NT) # is Windows_NT on XP, 2000, 7, Vista, 10... + detected_OS := Windows +else ifeq ($(strip $(shell uname)),Darwin) + detected_OS := macOS +else + # e.g. Linux + detected_OS := $(strip $(shell uname)) +endif + +# Builds and run a part of the test suite +test: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims + +# Builds and runs the smart contract tests +testContracts: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims + +# Builds and runs the integration tests +testIntegration: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim testIntegration $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims + +# Builds and runs all tests (except for Taiko L2 tests) +testAll: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims + +# Builds and runs Taiko L2 tests +testTaiko: | build deps + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims + +# Builds and runs tool tests +testTools: | cirdl + echo -e $(BUILD_MSG) "build/$@" && \ + $(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims + +# nim-libbacktrace +LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 +libbacktrace: +ifeq ($(detected_OS), Windows) +# MSYS2 detection +ifneq ($(MSYSTEM),) + + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) CMAKE_ARGS="-G'MSYS Makefiles'" +else + + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) +endif +else + + $(MAKE) $(LIBBACKTRACE_MAKE_FLAGS) +endif + +# usual cleaning +clean: | clean-common + rm -rf build +ifneq ($(USE_LIBBACKTRACE), 0) + + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) +endif + +############ +## Format ## +############ +.PHONY: build-nph install-nph-hook clean-nph print-nph-path + +# Default location for nph binary shall be next to nim binary to make it available on the path. +NPH:=$(shell dirname $(NIM_BINARY))/nph + +build-nph: +ifeq ("$(wildcard $(NPH))","") + $(ENV_SCRIPT) nim c vendor/nph/src/nph.nim && \ + mv vendor/nph/src/nph $(shell dirname $(NPH)) + echo "nph utility is available at " $(NPH) +endif + +GIT_PRE_COMMIT_HOOK := .git/hooks/pre-commit + +install-nph-hook: build-nph +ifeq ("$(wildcard $(GIT_PRE_COMMIT_HOOK))","") + cp ./tools/scripts/git_pre_commit_format.sh $(GIT_PRE_COMMIT_HOOK) +else + echo "$(GIT_PRE_COMMIT_HOOK) already present, will NOT override" + exit 1 +endif + +nph/%: build-nph + echo -e $(FORMAT_MSG) "nph/$*" && \ + $(NPH) $* + +format: + $(NPH) *.nim + $(NPH) codexcrawler/ + $(NPH) tests/ + +clean-nph: + rm -f $(NPH) + +# To avoid hardcoding nph binary location in several places +print-nph-path: + echo "$(NPH)" + +clean: | clean-nph + +endif # "variables.mk" was not included diff --git a/build.nims b/build.nims index 8866032..cfe8193 100644 --- a/build.nims +++ b/build.nims @@ -29,95 +29,15 @@ proc test(name: string, srcDir = "tests/", params = "", lang = "c") = buildBinary name, srcDir, params exec "build/" & name -task codex, "build codex binary": - buildBinary "codex", +task codexcrawler, "build codexcrawler binary": + buildBinary "codexcrawler", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" -task toolsCirdl, "build tools/cirdl binary": - buildBinary "tools/cirdl/cirdl" - -task testCodex, "Build & run Codex tests": - test "testCodex", params = "-d:codex_enable_proof_failures=true" - -task testContracts, "Build & run Codex Contract tests": - test "testContracts" - -task testIntegration, "Run integration tests": - buildBinary "codex", - params = - "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:codex_enable_proof_failures=true" - test "testIntegration" - # use params to enable logging from the integration test executable - # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & - # "-d:chronicles_enabled_topics:integration:TRACE" +task testCodexcrawler, "Build & run Codex Crawler tests": + test "testCodexCrawler", params = "-d:codex_enable_proof_failures=true" task build, "build codex binary": - codexTask() + codexCrawlerTask() task test, "Run tests": - testCodexTask() - -task testTools, "Run Tools tests": - toolsCirdlTask() - test "testTools" - -task testAll, "Run all tests (except for Taiko L2 tests)": - testCodexTask() - testContractsTask() - testIntegrationTask() - testToolsTask() - -task testTaiko, "Run Taiko L2 tests": - codexTask() - test "testTaiko" - -import strutils -import os - -task coverage, "generates code coverage report": - var (output, exitCode) = gorgeEx("which lcov") - if exitCode != 0: - echo " ************************** ⛔️ ERROR ⛔️ **************************" - echo " ** ERROR: lcov not found, it must be installed to run code **" - echo " ** coverage locally **" - echo " *****************************************************************" - quit 1 - - (output, exitCode) = gorgeEx("gcov --version") - if output.contains("Apple LLVM"): - echo " ************************* ⚠️ WARNING ⚠️ *************************" - echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **" - echo " ** emulates an old version of gcov (4.2.0) and therefore **" - echo " ** coverage results will differ than those on CI (which **" - echo " ** uses a much newer version of gcov). **" - echo " *****************************************************************" - - var nimSrcs = " " - for f in walkDirRec("codex", {pcFile}): - if f.endswith(".nim"): - nimSrcs.add " " & f.absolutePath.quoteShell() - - echo "======== Running Tests ======== " - test "coverage", - srcDir = "tests/", - params = - " --nimcache:nimcache/coverage -d:release -d:codex_enable_proof_failures=true" - exec("rm nimcache/coverage/*.c") - rmDir("coverage") - mkDir("coverage") - echo " ======== Running LCOV ======== " - exec( - "lcov --capture --keep-going --directory nimcache/coverage --output-file coverage/coverage.info" - ) - exec( - "lcov --extract coverage/coverage.info --keep-going --output-file coverage/coverage.f.info " & - nimSrcs - ) - echo " ======== Generating HTML coverage report ======== " - exec("genhtml coverage/coverage.f.info --keep-going --output-directory coverage/report ") - echo " ======== Coverage report Done ======== " - -task showCoverage, "open coverage html": - echo " ======== Opening HTML coverage report in browser... ======== " - if findExe("open") != "": - exec("open coverage/report/index.html") + testCodexCrawlerTask() diff --git a/vendor/nim-faststreams b/vendor/nim-faststreams new file mode 160000 index 0000000..c51315d --- /dev/null +++ b/vendor/nim-faststreams @@ -0,0 +1 @@ +Subproject commit c51315d0ae5eb2594d0bf41181d0e1aca1b3c01d diff --git a/nim-stew b/vendor/nim-stew similarity index 100% rename from nim-stew rename to vendor/nim-stew