diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fcb46df..c6d0bcc2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,7 +176,7 @@ jobs: - name: Run nim-libp2p tests run: | - nimble install -y --depsOnly + nimble install_pinned nimble test bumpNBC-stable: diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 8e4282a0c..f82646063 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -33,7 +33,7 @@ jobs: curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries export PATH="$PATH:$PWD/Nim/bin" - nimble install -y --depsOnly + nimble install_pinned export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}" nim c $NIM_OPTIONS -r ${{ matrix.test-program }} cd nimcache; rm *.c; cd .. @@ -71,8 +71,8 @@ jobs: curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries export PATH="$PATH:$PWD/Nim/bin" - nimble install -y --depsOnly - export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}" + nimble install_pinned + export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }} --clearNimblePath --NimblePath:nimbledeps/pkgs" nim c $NIM_OPTIONS -r ${{ matrix.test-program }} cd nimcache; rm *.c; cd .. lcov --capture --directory nimcache --output-file coverage/coverage.info @@ -112,7 +112,7 @@ jobs: curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries export PATH="$PATH:$PWD/Nim/bin" - nimble install -y --depsOnly + nimble install_pinned export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}" nim c $NIM_OPTIONS -r ${{ matrix.test-program }} cd nimcache; rm *.c; cd .. diff --git a/.gitignore b/.gitignore index 2ca1f19d3..e2d515b38 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ nimcache/ +nimbledeps/ # Executables shall be put in an ignored build/ directory # Ignore dynamic, static libs and libtool archive files diff --git a/.pinned b/.pinned new file mode 100644 index 000000000..5e19025c2 --- /dev/null +++ b/.pinned @@ -0,0 +1,17 @@ +asynctest;https://github.com/markspanbroek/asynctest@#3882ed64ed3159578f796bc5ae0c6b13837fe798 +bearssl;https://github.com/status-im/nim-bearssl@#ba80e2a0d7ae8aab666cee013e38ff8d33a3e5e7 +chronicles;https://github.com/status-im/nim-chronicles@#2a2681b60289aaf7895b7056f22616081eb1a882 +chronos;https://github.com/status-im/nim-chronos@#7dc58d42b6905a7fd7531875fa76060f8f744e4e +dnsclient;https://github.com/ba0f3/dnsclient.nim@#536cc6b7933e5f86590bb27083c0ffeab31255f9 +faststreams;https://github.com/status-im/nim-faststreams@#c653d05f277dca0f374732c5b9b80f2368faea33 +httputils;https://github.com/status-im/nim-http-utils@#507bfb7dcb6244d76ce2567df7bf3756cbe88775 +json_serialization;https://github.com/status-im/nim-json-serialization@#010aa238cf6afddf1fbe4cbcd27ab3be3f443841 +metrics;https://github.com/status-im/nim-metrics@#2c0c486c65f980e8387f86bed0b43d53161c8286 +nimcrypto;https://github.com/cheatfate/nimcrypto@#a5742a9a214ac33f91615f3862c7b099aec43b00 +secp256k1;https://github.com/status-im/nim-secp256k1@#d790c42206fab4b8008eaa91181ca8c8c68a0105 +serialization;https://github.com/status-im/nim-serialization@#11a8aa64d27d4fa92e266b9488500461da193c24 +stew;https://github.com/status-im/nim-stew@#2f9c61f485e1de6d7e163294008276c455d39da2 +testutils;https://github.com/status-im/nim-testutils@#aa6e5216f4b4ab5aa971cdcdd70e1ec1203cedf2 +unittest2;https://github.com/status-im/nim-unittest2@#b9b61cfe4a4b238de7175287b7481c35a798a582 +websock;https://github.com/status-im/nim-websock@#c2aae352f7fad7a8d333327c37e966969d3ee542 +zlib;https://github.com/status-im/nim-zlib@#d4e716d071eba1b5e0ffdf7949d983959e2b95dd \ No newline at end of file diff --git a/config.nims b/config.nims new file mode 100644 index 000000000..80f134395 --- /dev/null +++ b/config.nims @@ -0,0 +1,3 @@ +# to allow locking +if dirExists("nimbledeps/pkgs"): + switch("NimblePath", "nimbledeps/pkgs") diff --git a/libp2p.nimble b/libp2p.nimble index 04c55a96e..33be693a0 100644 --- a/libp2p.nimble +++ b/libp2p.nimble @@ -11,7 +11,7 @@ requires "nim >= 1.2.0", "nimcrypto >= 0.4.1", "https://github.com/ba0f3/dnsclient.nim == 0.1.0", "bearssl >= 0.1.4", - "chronicles#ba2817f1", + "chronicles >= 0.10.2", "chronos >= 3.0.6", "metrics", "secp256k1", @@ -94,3 +94,34 @@ task examples_build, "Build the samples": buildSample("helloworld", true) buildTutorial("examples/tutorial_1_connect.md") buildTutorial("examples/tutorial_2_customproto.md") + +# pin system +# while nimble lockfile +# isn't available + +const PinFile = ".pinned" +task pin, "Create a lockfile": + # pinner.nim was originally here + # but you can't read output from + # a command in a nimscript + exec "nim c -r tools/pinner.nim" + +import sequtils +import os +task install_pinned, "Reads the lockfile": + let toInstall = readFile(PinFile).splitWhitespace().mapIt((it.split(";", 1)[0], it.split(";", 1)[1])) + # [('packageName', 'packageFullUri')] + + rmDir("nimbledeps") + mkDir("nimbledeps") + exec "nimble install -y " & toInstall.mapIt(it[1]).join(" ") + + # Remove the automatically installed deps + # (inefficient you say?) + let allowedDirectories = toInstall.mapIt(it[0] & "-" & it[1].split('@')[1]) + for dependency in listDirs("nimbledeps/pkgs"): + if dependency.extractFilename notin allowedDirectories: + rmDir(dependency) + +task unpin, "Restore global package use": + rmDir("nimbledeps") diff --git a/tools/pinner.nim b/tools/pinner.nim new file mode 100644 index 000000000..702870d39 --- /dev/null +++ b/tools/pinner.nim @@ -0,0 +1,38 @@ +import os +import strscans +import algorithm +import tables +import sequtils +import strutils +import json +import osproc + +const PinFile = ".pinned" +removeDir("nimbledeps") +createDir("nimbledeps") +discard execCmd("nimble install -dy") + +var allDeps: Table[string, string] +for (_, dependency) in walkDir("nimbledeps/pkgs"): + let fileContent = parseJson(readFile(dependency & "/nimblemeta.json")) + let url = fileContent.getOrDefault("url").getStr("") + var version = fileContent.getOrDefault("vcsRevision").getStr("") + var packageName = dependency.split('/')[^1].split('-')[0] + + if version.len == 0: + version = execCmdEx("git ls-remote " & url).output.split()[0] + + if version.len > 0 and url.len > 0: + let fullPackage = url & "@#" & version + if packageName in allDeps and allDeps[packageName] != fullPackage: + echo "Warning: duplicate package " & packageName & ":" + echo allDeps[packageName] + echo fullPackage + echo "" + allDeps[packageName] = fullPackage + else: + echo "Failed to get url & version for ", dependency + +let asString = toSeq(allDeps.pairs).mapIt(it[0] & ";" & it[1]).sorted().join("\n") +writeFile(PinFile, asString) +echo asString