Merge pull request #67 from codex-storage/nimble-build-light

* import nimbus build tools
* add nimbus build
* updates
* add nat traversal to make nimbus happy
* clone nimbus vendor using atlas
* disable ci-nimbus
* only run ci-nimbus on lockfile change
* add nim setup
* caching - change names
* restore names
* cleanup & test
* bump version
This commit is contained in:
Jaremy Creechley 2023-07-12 14:53:52 -07:00 committed by GitHub
commit 96515d6d45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 496 additions and 75 deletions

173
.github/workflows/ci-nimbus.yml vendored Normal file
View File

@ -0,0 +1,173 @@
name: CI-nimbus
on:
push:
paths:
- atlas.lock
- .github/workflows/ci-nimbus.yml
jobs:
build:
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
target:
- os: linux
cpu: amd64
# - os: linux
# cpu: i386
- os: macos
cpu: amd64
- os: windows
cpu: amd64
#- os: windows
#cpu: i386
branch: [version-1-6]
include:
- target:
os: linux
builder: ubuntu-20.04
shell: bash
- target:
os: macos
builder: macos-12
shell: bash
- target:
os: windows
builder: windows-2019
shell: msys2 {0}
defaults:
run:
shell: ${{ matrix.shell }}
name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
runs-on: ${{ matrix.builder }}
continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: true
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: MSYS2 (Windows i386)
if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: MINGW32
install: >-
base-devel
git
mingw-w64-i686-toolchain
- name: MSYS2 (Windows amd64)
if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
install: >-
base-devel
git
mingw-w64-x86_64-toolchain
- name: Restore Nim DLLs dependencies (Windows) from cache
if: runner.os == 'Windows'
id: windows-dlls-cache
uses: actions/cache@v2
with:
path: external/dlls
key: 'dlls'
- name: Install DLL dependencies (Windows)
if: >
steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
runner.os == 'Windows'
run: |
mkdir external
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x external/windeps.zip -oexternal/dlls
- name: Path to cached dependencies (Windows)
if: >
runner.os == 'Windows'
run: |
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Derive environment variables
run: |
if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
PLATFORM=x64
else
PLATFORM=x86
fi
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
ncpu=
MAKE_CMD="make"
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
MAKE_CMD="mingw32-make"
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: 1.6.14
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Restore Nim toolchain binaries from cache
id: nim-cache
uses: actions/cache@v3
with:
path: NimBinaries
key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nim-${{ hashFiles('atlas.lock') }}
- name: Restore Vendor Clones from cache
id: vendor-cache
uses: actions/cache@v3
with:
path: vendor/*/
key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-vendor-${{ hashFiles('atlas.lock') }}
- name: Run tests
run: |
if [[ "${{ matrix.target.os }}" == "windows" ]]; then
# https://github.com/status-im/nimbus-eth2/issues/3121
export NIMFLAGS="-d:nimRawSetjmp"
fi
echo "BUILD: "
export NIM_COMMIT=${{ matrix.branch }}
make -j${ncpu} CI_CACHE=NimBinaries ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1
make test -j${ncpu}

View File

@ -140,14 +140,14 @@ jobs:
echo "ncpu=$ncpu" >> $GITHUB_ENV
echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
# - name: Restore nimble dependencies from cache
# id: nimble_deps
# uses: actions/cache@v3
# with:
# path: |
# ~/.nimble/pkgs2
# ~/.nimble/packages_official.json
# key: ${{ inputs.os }}-${{ inputs.cpu }}-${{ env.cache_nonce }}
- name: Restore nimble dependencies from cache
id: nimble_deps
uses: actions/cache@v3
with:
path: |
~/.nimble/pkgs2
~/.nimble/packages_official.json
key: ${{ matrix.target.os }}-${{ matrix.target.cpu }}-nimble-${{ hashFiles('nimble.lock') }}
- name: Setup Nimble
uses: "./.github/actions/install_nimble"

5
.gitignore vendored
View File

@ -3,3 +3,8 @@ nimcache
tests/testAll
nimble.develop
nimble.paths
nim.cfg
nimbus-build-system.paths
vendor/*
NimBinaries
.update.timestamp

71
Makefile Normal file
View File

@ -0,0 +1,71 @@
# Copyright (c) 2020 Status Research & Development GmbH. Licensed under
# either of:
# - Apache License, version 2.0
# - MIT license
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
SHELL := bash # the shell used internally by Make
# used inside the included makefiles
BUILD_SYSTEM_DIR := vendor/nimbus-build-system
# -d:insecure - Necessary to enable Prometheus HTTP endpoint for metrics
# -d:chronicles_colors:none - Necessary to disable colors in logs for Docker
DOCKER_IMAGE_NIM_PARAMS ?= -d:chronicles_colors:none -d:insecure
LINK_PCRE := 0
# we don't want an error here, so we can handle things later, in the ".DEFAULT" target
-include $(BUILD_SYSTEM_DIR)/makefiles/variables.mk
.PHONY: \
all \
clean \
coverage \
deps \
libbacktrace \
test \
update
ifeq ($(NIM_PARAMS),)
# "variables.mk" was not included, so we update the submodules.
GIT_SUBMODULE_UPDATE := nimble install https://github.com/elcritch/atlas && atlas rep --noexec atlas.lock
.DEFAULT:
+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
$(GIT_SUBMODULE_UPDATE); \
echo
# Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
# https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles
#
# After restarting, it will execute its original goal, so we don't have to start a child Make here
# with "$(MAKE) $(MAKECMDGOALS)". Isn't hidden control flow great?
else # "variables.mk" was included. Business as usual until the end of this file.
# default target, because it's the first one that doesn't start with '.'
# Builds the codex binary
all: | build deps
echo -e $(BUILD_MSG) "$@" && \
$(ENV_SCRIPT) nim test $(NIM_PARAMS)
# must be included after the default target
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
deps: | deps-common nat-libs
#- deletes and recreates "codexdht.nims" which on Windows is a copy instead of a proper symlink
update: | update-common
rm -rf codexdht.nims && \
$(MAKE) codexdht.nims $(HANDLE_OUTPUT)
# Builds and run a part of the test suite
test: | build deps
echo -e $(BUILD_MSG) "$@" && \
$(ENV_SCRIPT) nim testAll $(NIM_PARAMS) config.nims
# usual cleaning
clean: | clean-common
endif # "variables.mk" was not included

143
atlas.lock Normal file
View File

@ -0,0 +1,143 @@
{
"items": {
"nimbus-build-system": {
"dir": "vendor/nimbus-build-system",
"url": "https://github.com/status-im/nimbus-build-system",
"commit": "239c3a7fbb88fd241da0ade3246fd2e5fcff4f25"
},
"nim-nat-traversal": {
"dir": "vendor/nim-nat-traversal",
"url": "https://github.com/status-im/nim-nat-traversal",
"commit": "802d75edcc656e616120fb27f950ff1285ddcbba"
},
"nim-zlib": {
"dir": "vendor/nim-zlib",
"url": "https://github.com/status-im/nim-zlib",
"commit": "f34ca261efd90f118dc1647beefd2f7a69b05d93"
},
"nim-stew": {
"dir": "vendor/nim-stew",
"url": "https://github.com/status-im/nim-stew.git",
"commit": "e18f5a62af2ade7a1fd1d39635d4e04d944def08"
},
"nim-http-utils": {
"dir": "vendor/nim-http-utils",
"url": "https://github.com/status-im/nim-http-utils.git",
"commit": "3b491a40c60aad9e8d3407443f46f62511e63b18"
},
"nim-chronos": {
"dir": "vendor/nim-chronos",
"url": "https://github.com/status-im/nim-chronos.git",
"commit": "6525f4ce1d1a7eba146e5f1a53f6f105077ae686"
},
"upraises": {
"dir": "vendor/upraises",
"url": "https://github.com/markspanbroek/upraises.git",
"commit": "bc2628989b63854d980e92dadbd58f83e34b6f25"
},
"nim-sqlite3-abi": {
"dir": "vendor/nim-sqlite3-abi",
"url": "https://github.com/arnetheduck/nim-sqlite3-abi.git",
"commit": "362e1bd9f689ad9f5380d9d27f0705b3d4dfc7d3"
},
"questionable": {
"dir": "vendor/questionable",
"url": "https://github.com/status-im/questionable.git",
"commit": "0d7ce8efdedaf184680cb7268721fca0af947a74"
},
"nim-websock": {
"dir": "vendor/nim-websock",
"url": "https://github.com/status-im/nim-websock.git",
"commit": "2c3ae3137f3c9cb48134285bd4a47186fa51f0e8"
},
"nim-secp256k1": {
"dir": "vendor/nim-secp256k1",
"url": "https://github.com/status-im/nim-secp256k1.git",
"commit": "5340cf188168d6afcafc8023770d880f067c0b2f"
},
"nim-bearssl": {
"dir": "vendor/nim-bearssl",
"url": "https://github.com/status-im/nim-bearssl.git",
"commit": "f4c4233de453cb7eac0ce3f3ffad6496295f83ab"
},
"dnsclient.nim": {
"dir": "vendor/dnsclient.nim",
"url": "https://github.com/ba0f3/dnsclient.nim",
"commit": "23214235d4784d24aceed99bbfe153379ea557c8"
},
"nimcrypto": {
"dir": "vendor/nimcrypto",
"url": "https://github.com/status-im/nimcrypto.git",
"commit": "a5742a9a214ac33f91615f3862c7b099aec43b00"
},
"nim-json-serialization": {
"dir": "vendor/nim-json-serialization",
"url": "https://github.com/status-im/nim-json-serialization.git",
"commit": "e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4"
},
"nim-testutils": {
"dir": "vendor/nim-testutils",
"url": "https://github.com/status-im/nim-testutils",
"commit": "b56a5953e37fc5117bd6ea6dfa18418c5e112815"
},
"nim-unittest2": {
"dir": "vendor/nim-unittest2",
"url": "https://github.com/status-im/nim-unittest2.git",
"commit": "b178f47527074964f76c395ad0dfc81cf118f379"
},
"npeg": {
"dir": "vendor/npeg",
"url": "https://github.com/zevv/npeg",
"commit": "b15a10e388b91b898c581dbbcb6a718d46b27d2f"
},
"nim-serialization": {
"dir": "vendor/nim-serialization",
"url": "https://github.com/status-im/nim-serialization.git",
"commit": "493d18b8292fc03aa4f835fd825dea1183f97466"
},
"nim-faststreams": {
"dir": "vendor/nim-faststreams",
"url": "https://github.com/status-im/nim-faststreams.git",
"commit": "1b561a9e71b6bdad1c1cdff753418906037e9d09"
},
"nim-datastore": {
"dir": "vendor/nim-datastore",
"url": "https://github.com/codex-storage/nim-datastore.git",
"commit": "0cde8aeb67c59fd0ac95496dc6b5e1168d6632aa"
},
"asynctest": {
"dir": "vendor/asynctest",
"url": "https://github.com/markspanbroek/asynctest",
"commit": "a236a5f0f3031573ac2cb082b63dbf6e170e06e7"
},
"nim-stint": {
"dir": "vendor/nim-stint",
"url": "https://github.com/status-im/nim-stint.git",
"commit": "036c71d06a6b22f8f967ba9d54afd2189c3872ca"
},
"nim-metrics": {
"dir": "vendor/nim-metrics",
"url": "https://github.com/status-im/nim-metrics.git",
"commit": "743f81d4f6c6ebf0ac02389f2392ff8b4235bee5"
},
"nim-libp2p": {
"dir": "vendor/nim-libp2p",
"url": "https://github.com/status-im/nim-libp2p.git",
"commit": "a3e9d1ed80c048cd5abc839cbe0863cefcedc702"
},
"nim-chronicles": {
"dir": "vendor/nim-chronicles",
"url": "https://github.com/status-im/nim-chronicles.git",
"commit": "7631f7b2ee03398cb1512a79923264e8f9410af6"
},
"nim-protobuf-serialization": {
"dir": "vendor/nim-protobuf-serialization",
"url": "https://github.com/status-im/nim-protobuf-serialization",
"commit": "28214b3e40c755a9886d2ec8f261ec48fbb6bec6"
}
},
"nimcfg": "############# begin Atlas config section ##########\n--noNimblePath\n--path:\"vendor/nim-secp256k1\"\n--path:\"vendor/nim-protobuf-serialization\"\n--path:\"vendor/nimcrypto\"\n--path:\"vendor/nim-bearssl\"\n--path:\"vendor/nim-chronicles\"\n--path:\"vendor/nim-chronos\"\n--path:\"vendor/nim-libp2p\"\n--path:\"vendor/nim-metrics\"\n--path:\"vendor/nim-stew\"\n--path:\"vendor/nim-stint\"\n--path:\"vendor/asynctest\"\n--path:\"vendor/nim-datastore\"\n--path:\"vendor/questionable\"\n--path:\"vendor/nim-faststreams\"\n--path:\"vendor/nim-serialization\"\n--path:\"vendor/npeg/src\"\n--path:\"vendor/nim-unittest2\"\n--path:\"vendor/nim-testutils\"\n--path:\"vendor/nim-json-serialization\"\n--path:\"vendor/nim-http-utils\"\n--path:\"vendor/dnsclient.nim/src\"\n--path:\"vendor/nim-websock\"\n--path:\"vendor/nim-sqlite3-abi\"\n--path:\"vendor/upraises\"\n--path:\"vendor/nim-zlib\"\n############# end Atlas config section ##########\n",
"nimVersion": "1.6.14",
"gccVersion": "",
"clangVersion": ""
}

65
codexdht.nimble Normal file
View File

@ -0,0 +1,65 @@
# Package
version = "0.2.1"
author = "Status Research & Development GmbH"
description = "DHT based on the libp2p Kademlia spec"
license = "MIT"
skipDirs = @["tests"]
# Dependencies
requires "nim >= 1.2.0"
requires "secp256k1#b3f38e2795e805743b299dc5d96d332db375b520" # >= 0.5.2 & < 0.6.0
requires "protobuf_serialization#27b400fdf3bd8ce7120ca66fc1de39d3f1a5804a" # >= 0.2.0 & < 0.3.0
requires "nimcrypto == 0.5.4"
requires "bearssl#head"
requires "chronicles >= 0.10.2 & < 0.11.0"
requires "chronos#1394c9e04957928afc1db33d2e0965cfb677a1e0" # >= 3.0.11 & < 3.1.0
requires "libp2p#unstable"
requires "metrics"
requires "stew#head"
requires "stint"
requires "asynctest >= 0.3.1 & < 0.4.0"
requires "https://github.com/status-im/nim-datastore#head"
requires "questionable"
task testAll, "Run DHT tests":
exec "nim c -r tests/testAll.nim"
# task coverage, "generates code coverage report":
# var (output, exitCode) = gorgeEx("which lcov")
# if exitCode != 0:
# echo ""
# echo " ************************** ⛔️ ERROR ⛔️ **************************"
# echo " ** **"
# echo " ** ERROR: lcov not found, it must be installed to run code **"
# echo " ** coverage locally **"
# echo " ** **"
# echo " *****************************************************************"
# echo ""
# quit 1
# (output, exitCode) = gorgeEx("gcov --version")
# if output.contains("Apple LLVM"):
# echo ""
# echo " ************************* ⚠️ WARNING ⚠️ *************************"
# echo " ** **"
# echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **"
# echo " ** emulates an old version of gcov (4.2.0) and therefore **"
# echo " ** coverage results will differ than those on CI (which **"
# echo " ** uses a much newer version of gcov). **"
# echo " ** **"
# echo " *****************************************************************"
# echo ""
# exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage")
# exec("cd nimcache; rm *.c; cd ..")
# mkDir("coverage")
# exec("lcov --capture --directory nimcache --output-file coverage/coverage.info")
# exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'")
# exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'")
# echo "Generating HTML coverage report"
# exec("genhtml coverage/coverage.f.info --output-directory coverage/report")
# echo "Opening HTML coverage report in browser..."
# exec("open coverage/report/index.html")

View File

@ -1,7 +1,25 @@
import std/os
const currentDir = currentSourcePath()[0 .. ^(len("config.nims") + 1)]
switch("define", "libp2p_pki_schemes=secp256k1")
# begin Nimble config (version 2)
--noNimblePath
when withDir(thisDir(), system.fileExists("nimble.paths")):
task testAll, "Run DHT tests":
exec "nim c -r tests/testAll.nim"
task test, "Run DHT tests":
testAllTask()
when getEnv("NIMBUS_BUILD_SYSTEM") == "yes" and
# BEWARE
# In Nim 1.6, config files are evaluated with a working directory
# matching where the Nim command was invocated. This means that we
# must do all file existance checks with full absolute paths:
system.fileExists(currentDir & "nimbus-build-system.paths"):
echo "Using Nimbus Paths"
include "nimbus-build-system.paths"
elif fileExists("nimble.paths"):
echo "Using Nimble Paths"
# begin Nimble config (version 1)
include "nimble.paths"
# end Nimble config
# end Nimble config

7
env.sh Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
# We use ${BASH_SOURCE[0]} instead of $0 to allow sourcing this file
# and we fall back to a Zsh-specific special var to also support Zsh.
REL_PATH="$(dirname ${BASH_SOURCE[0]:-${(%):-%x}})"
ABS_PATH="$(cd ${REL_PATH}; pwd)"
source ${ABS_PATH}/vendor/nimbus-build-system/scripts/env.sh

View File

@ -1,63 +0,0 @@
# Package
version = "0.2.0"
author = "Status Research & Development GmbH"
description = "DHT based on the libp2p Kademlia spec"
license = "MIT"
skipDirs = @["tests"]
# TODO: fix versions after tagging the version in the corresponding package
# Dependencies
requires "nim >= 1.2.0"
requires "secp256k1#b3f38e2795e805743b299dc5d96d332db375b520" # >= 0.5.2 & < 0.6.0
requires "protobuf_serialization#27b400fdf3bd8ce7120ca66fc1de39d3f1a5804a" # >= 0.2.0 & < 0.3.0
requires "nimcrypto == 0.5.4"
requires "bearssl#head"
requires "chronicles >= 0.10.2 & < 0.11.0"
requires "chronos#1394c9e04957928afc1db33d2e0965cfb677a1e0" # >= 3.0.11 & < 3.1.0
requires "libp2p#unstable"
requires "metrics"
requires "stew#head"
requires "stint"
requires "asynctest >= 0.3.1 & < 0.4.0"
requires "https://github.com/status-im/nim-datastore#head"
requires "questionable"
task coverage, "generates code coverage report":
var (output, exitCode) = gorgeEx("which lcov")
if exitCode != 0:
echo ""
echo " ************************** ⛔️ ERROR ⛔️ **************************"
echo " ** **"
echo " ** ERROR: lcov not found, it must be installed to run code **"
echo " ** coverage locally **"
echo " ** **"
echo " *****************************************************************"
echo ""
quit 1
(output, exitCode) = gorgeEx("gcov --version")
if output.contains("Apple LLVM"):
echo ""
echo " ************************* ⚠️ WARNING ⚠️ *************************"
echo " ** **"
echo " ** WARNING: Using Apple's llvm-cov in place of gcov, which **"
echo " ** emulates an old version of gcov (4.2.0) and therefore **"
echo " ** coverage results will differ than those on CI (which **"
echo " ** uses a much newer version of gcov). **"
echo " ** **"
echo " *****************************************************************"
echo ""
exec("nimble --verbose test --opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage")
exec("cd nimcache; rm *.c; cd ..")
mkDir("coverage")
exec("lcov --capture --directory nimcache --output-file coverage/coverage.info")
exec("$(which bash) -c 'shopt -s globstar; ls $(pwd)/libp2pdht/{*,**/*}.nim'")
exec("$(which bash) -c 'shopt -s globstar; lcov --extract coverage/coverage.info $(pwd)/libp2pdht/{*,**/*}.nim --output-file coverage/coverage.f.info'")
echo "Generating HTML coverage report"
exec("genhtml coverage/coverage.f.info --output-directory coverage/report")
echo "Opening HTML coverage report in browser..."
exec("open coverage/report/index.html")

2
vendor/atlas.workspace vendored Normal file
View File

@ -0,0 +1,2 @@
deps=""
resolver="MaxVer"