normalise nimble, ci (#567)

* normalise nimble, ci

* drop obsolete dep download

* add threads

.github/workflows/ci.yml

@@ -6,6 +6,10 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency: # Cancel stale PR builds (but not push builds)
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
 jobs:
   build:
     strategy:
@@ -69,64 +73,6 @@ jobs:
           chmod 755 external/bin/gcc external/bin/g++
           echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
 
-      - name: 'Install dependencies (macOS)'
-        if: runner.os == 'macOS' && matrix.branch == 'devel'
-        run: |
-          brew install openssl@1.1
-          ln -s $(brew --prefix)/opt/openssl/lib/libcrypto.1.1.dylib /usr/local/lib/
-          ln -s $(brew --prefix)/opt/openssl/lib/libssl.1.1.dylib /usr/local/lib/
-
-      - name: Restore rocksdb from cache (Macos)
-        if: runner.os != 'Windows'
-        id: rocksdb-cache
-        uses: actions/cache@v3
-        with:
-          path: rocks-db-cache-${{ matrix.target.cpu }}
-          key: 'rocksdb-v1-${{ matrix.target.os }}-${{ matrix.target.cpu }}'
-
-      - name: Build and install rocksdb (Linux i386)
-        # no librocksdb-dev:i386
-        if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
-        run: |
-          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_rocksdb.sh
-          bash build_rocksdb.sh rocks-db-cache-${{ matrix.target.cpu }}
-
-      - name: Install rocksdb (Linux amd64)
-        # mysterious illegal instruction error if we build our own librocksdb
-        if: runner.os == 'Linux' && matrix.target.cpu == 'amd64'
-        run: |
-         sudo apt-get -q update
-         sudo apt-get install -y librocksdb-dev
-
-      - name: Build and install lmdb (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          LMDBVER="0.9.22"
-          curl -L "https://github.com/LMDB/lmdb/archive/LMDB_$LMDBVER.tar.gz" -o "LMDB_$LMDBVER.tar.gz"
-          tar xzf "LMDB_$LMDBVER.tar.gz"
-          cd "lmdb-LMDB_$LMDBVER/libraries/liblmdb"
-          make -j2
-          sed -i 's| liblmdb.a||' Makefile
-          sudo make prefix=/usr install
-
-      - name: Build and install rocksdb (Macos)
-        if: runner.os == 'Macos'
-        run: |
-          HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 brew install ccache
-          echo "/usr/local/opt/ccache/libexec" >> $GITHUB_PATH
-          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_rocksdb.sh
-          bash build_rocksdb.sh rocks-db-cache-${{ matrix.target.cpu }}
-
-      - name: Build and install lmdb (Macos)
-        if: runner.os == 'Macos'
-        run: |
-          LMDBVER="0.9.22"
-          curl -L "https://github.com/LMDB/lmdb/archive/LMDB_$LMDBVER.tar.gz" -o "LMDB_$LMDBVER.tar.gz"
-          tar xzf "LMDB_$LMDBVER.tar.gz"
-          cd "lmdb-LMDB_$LMDBVER/libraries/liblmdb"
-          make -j2
-          sudo cp -a liblmdb.so /usr/local/lib/liblmdb.dylib
-
       - name: MSYS2 (Windows i386)
         if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
         uses: msys2/setup-msys2@v2
@@ -161,42 +107,15 @@ jobs:
           steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
           runner.os == 'Windows'
         run: |
-          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
-            ROCKSDBSUB=x64
-            SQLITE_URL="https://www.sqlite.org/2018/sqlite-dll-win64-x64-3240000.zip"
-            SQLITE_DLL="sqlite3_64.dll"
-          else
-            ROCKSDBSUB=x86
-            SQLITE_URL="https://www.sqlite.org/2018/sqlite-dll-win32-x86-3240000.zip"
-            SQLITE_DLL="sqlite3_32.dll"
-          fi
-          DLLPATH=external/dlls-${{ matrix.target.cpu }}
           mkdir -p external
           curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
-          7z x -y external/windeps.zip -o"$DLLPATH"
-          # ROCKSDB
-          curl -L "https://github.com/status-im/nimbus-deps/releases/download/nimbus-deps/nimbus-deps.zip" -o external/nimbus-deps.zip
-          7z x -y external/nimbus-deps.zip
-          cp "./$ROCKSDBSUB/librocksdb.dll" "$DLLPATH/librocksdb.dll"
-          # SQLITE
-          curl -L "$SQLITE_URL" -o external/sqlite-dll.zip
-          7z x -y external/sqlite-dll.zip
-          cp ./sqlite3.dll "$DLLPATH/$SQLITE_DLL"
-          # LMDB
-          LMDBVER="0.9.22"
-          LMDB_PATH="lmdb-LMDB_$LMDBVER/libraries/liblmdb"
-          curl -L "https://github.com/LMDB/lmdb/archive/LMDB_$LMDBVER.tar.gz" -o "LMDB_$LMDBVER.tar.gz"
-          tar xzf "LMDB_$LMDBVER.tar.gz"
-          cd "$LMDB_PATH"
-          gcc -march=native -O3 -pipe -shared -o lmdb.dll mdb.c midl.c -lntdll
-          cd ../../..
-          cp "$LMDB_PATH/lmdb.dll" "$DLLPATH/lmdb.dll"
+          7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }}
 
       - name: Path to cached dependencies (Windows)
         if: >
           runner.os == 'Windows'
         run: |
-          echo '${{ github.workspace }}'"/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH
+          echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH
 
       - name: Derive environment variables
         run: |
@@ -236,7 +155,6 @@ jobs:
       - name: Run tests
         run: |
           export PLATFORM="${{ matrix.target.os }}-${{ matrix.target.cpu }}"
-          export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib"
           nim --version
           nimble --version
           nimble install -y

.gitignore

@@ -5,6 +5,7 @@
 !LICENSE*
 
 nimcache/
+build/
 
 # Executables shall be put in an ignored build/ directory
 # Ignore dynamic, static libs and libtool archive files

eth.nimble

@@ -1,3 +1,5 @@
+mode = ScriptMode.Verbose
+
 version       = "1.0.0"
 author        = "Status Research & Development GmbH"
 description   = "Ethereum Common library"
@@ -18,71 +20,65 @@ requires "nim >= 1.6.0",
          "testutils",
          "unittest2"
 
-let commonParams =
-  " --skipUserCfg:on" &
-  " --verbosity:0 --hints:off" &
-  " --warning[ObservableStores]:off" &
+let nimc = getEnv("NIMC", "nim") # Which nim compiler to use
+let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js)
+let flags = getEnv("NIMFLAGS", "") # Extra flags for the compiler
+let verbose = getEnv("V", "") notin ["", "0"]
+
+let cfg =
   " --styleCheck:usages --styleCheck:error" &
-  " " & getEnv("NIMFLAGS") &
-  " -d:chronicles_log_level=TRACE"
+  (if verbose: "" else: " --verbosity:0 --hints:off") &
+  " --skipUserCfg --nimcache:build/nimcache -f" &
+  " --warning[ObservableStores]:off -d:nimOldCaseObjects" &
+  " -d:chronosStrictException -d:chronicles_log_level=TRACE" &
+  " --threads:on"
 
-proc runTest(path: string, release: bool = true) =
-  echo "\nBuilding and running: ", path
-  let releaseMode = if release: " -d:release" else: ""
+# Windows CI releaseMode="-d:debug" or releaseMode=""
+# will cause mysterious crash
+let releaseMode = "-d:release"
 
-  exec "nim c -r" &
-    releaseMode & commonParams & " " & path
-  rmFile path
+proc build(args, path, outdir: string) =
+  exec nimc & " " & lang & " " & cfg & " " & flags & " " & args &
+    " --outdir:build/" & outdir & " " & path
 
-proc buildBinary(path: string) =
-  echo "\nBuilding: ", path
-  exec "nim c -d:release" & commonParams &
-    " --warning[CaseTransition]:off" &
-    " " & path
+proc run(args, path, outdir: string) =
+  build args & " -r", path, outdir
 
 task test_keyfile, "Run keyfile tests":
-  runTest("tests/keyfile/all_tests")
+  run "-d:release", "tests/keyfile/all_tests", "keyfile"
 
 task test_keys, "Run keys tests":
-  runTest("tests/keys/all_tests")
+  run "-d:release", "tests/keys/all_tests", "keys"
 
 task test_discv5, "Run discovery v5 tests":
-  runTest("tests/p2p/all_discv5_tests")
+  run "-d:release", "tests/p2p/all_discv5_tests", "p2p"
 
 task test_discv4, "Run discovery v4 tests":
-  runTest("tests/p2p/test_discovery")
+  run "-d:release", "tests/p2p/test_discovery", "p2p"
 
 task test_p2p, "Run p2p tests":
-  runTest("tests/p2p/all_tests")
+  run releaseMode, "tests/p2p/all_tests", "p2p"
 
 task test_rlp, "Run rlp tests":
-  # workaround for github action CI
-  # mysterious crash on windows-2019 64bit mode
-  # cannot reproduce locally on windows-2019
-  # running in virtualbox
-  let releaseMode = if existsEnv"PLATFORM":
-                      getEnv"PLATFORM" != "windows-amd64"
-                    else: true
-
-  runTest("tests/rlp/all_tests", releaseMode)
+  run releaseMode, "tests/rlp/all_tests", "rlp"
 
 task test_trie, "Run trie tests":
-  runTest("tests/trie/all_tests")
+  run "-d:release", "tests/trie/all_tests", "trie"
 
 task test_db, "Run db tests":
-  runTest("tests/db/all_tests")
+  run "-d:release", "tests/db/all_tests", "db"
 
 task test_utp, "Run utp tests":
-  runTest("tests/utp/all_utp_tests")
+  run "-d:release", "tests/utp/all_utp_tests", "utp"
 
 task test_common, "Run common tests":
-  runTest("tests/common/all_tests")
+  run "-d:release", "tests/common/all_tests", "common"
 
 task test, "Run all tests":
   for filename in [
       "test_bloom",
     ]:
-    runTest("tests/" & filename)
+    run "-d:release", "tests/" & filename, ""
 
   test_keyfile_task()
   test_keys_task()
@@ -99,7 +95,7 @@ task test_discv5_full, "Run discovery v5 and its dependencies tests":
   test_discv5_task()
 
 task build_dcli, "Build dcli":
-  buildBinary("tools/dcli")
+  build "-d:release", "tools/dcli",""
 
 import os, strutils
 
@@ -112,4 +108,4 @@ task build_fuzzers, "Build fuzzer test cases":
   # https://github.com/status-im/nim-testutils/blob/master/testutils/fuzzing.nim#L100
   for file in walkDirRec("tests/fuzzing/"):
     if file.endsWith("nim"):
-      buildBinary(file)
+      build "", file, "fuzzing"