Pin dependencies for CI (#677)

Adds a pin-file for CI. When a dependency break something in the libp2p, we don't want for the CI to be
broken on every branch, just on a "bump" branch that will update the pin-file & fix the eventual interop issue.

.github/workflows/ci.yml

@@ -176,7 +176,7 @@ jobs:
 
       - name: Run nim-libp2p tests
         run: |
-          nimble install -y --depsOnly
+          nimble install_pinned
           nimble test
 
   bumpNBC-stable:

.github/workflows/codecov.yml

@@ -33,7 +33,7 @@ jobs:
           curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
           env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
           export PATH="$PATH:$PWD/Nim/bin"
-          nimble install -y --depsOnly
+          nimble install_pinned
           export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
           nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
           cd nimcache; rm *.c; cd ..
@@ -71,8 +71,8 @@ jobs:
           curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
           env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
           export PATH="$PATH:$PWD/Nim/bin"
-          nimble install -y --depsOnly
+          nimble install_pinned
-          export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
+          export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }} --clearNimblePath --NimblePath:nimbledeps/pkgs"
           nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
           cd nimcache; rm *.c; cd ..
           lcov --capture --directory nimcache --output-file coverage/coverage.info
@@ -112,7 +112,7 @@ jobs:
           curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
           env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
           export PATH="$PATH:$PWD/Nim/bin"
-          nimble install -y --depsOnly
+          nimble install_pinned
           export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
           nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
           cd nimcache; rm *.c; cd ..

.gitignore

@@ -1,4 +1,5 @@
 nimcache/
+nimbledeps/
 
 # Executables shall be put in an ignored build/ directory
 # Ignore dynamic, static libs and libtool archive files

.pinned

@@ -0,0 +1,17 @@
+asynctest;https://github.com/markspanbroek/asynctest@#3882ed64ed3159578f796bc5ae0c6b13837fe798
+bearssl;https://github.com/status-im/nim-bearssl@#ba80e2a0d7ae8aab666cee013e38ff8d33a3e5e7
+chronicles;https://github.com/status-im/nim-chronicles@#2a2681b60289aaf7895b7056f22616081eb1a882
+chronos;https://github.com/status-im/nim-chronos@#7dc58d42b6905a7fd7531875fa76060f8f744e4e
+dnsclient;https://github.com/ba0f3/dnsclient.nim@#536cc6b7933e5f86590bb27083c0ffeab31255f9
+faststreams;https://github.com/status-im/nim-faststreams@#c653d05f277dca0f374732c5b9b80f2368faea33
+httputils;https://github.com/status-im/nim-http-utils@#507bfb7dcb6244d76ce2567df7bf3756cbe88775
+json_serialization;https://github.com/status-im/nim-json-serialization@#010aa238cf6afddf1fbe4cbcd27ab3be3f443841
+metrics;https://github.com/status-im/nim-metrics@#2c0c486c65f980e8387f86bed0b43d53161c8286
+nimcrypto;https://github.com/cheatfate/nimcrypto@#a5742a9a214ac33f91615f3862c7b099aec43b00
+secp256k1;https://github.com/status-im/nim-secp256k1@#d790c42206fab4b8008eaa91181ca8c8c68a0105
+serialization;https://github.com/status-im/nim-serialization@#11a8aa64d27d4fa92e266b9488500461da193c24
+stew;https://github.com/status-im/nim-stew@#2f9c61f485e1de6d7e163294008276c455d39da2
+testutils;https://github.com/status-im/nim-testutils@#aa6e5216f4b4ab5aa971cdcdd70e1ec1203cedf2
+unittest2;https://github.com/status-im/nim-unittest2@#b9b61cfe4a4b238de7175287b7481c35a798a582
+websock;https://github.com/status-im/nim-websock@#c2aae352f7fad7a8d333327c37e966969d3ee542
+zlib;https://github.com/status-im/nim-zlib@#d4e716d071eba1b5e0ffdf7949d983959e2b95dd

config.nims

@@ -0,0 +1,3 @@
+# to allow locking
+if dirExists("nimbledeps/pkgs"):
+  switch("NimblePath", "nimbledeps/pkgs")

libp2p.nimble

@@ -11,7 +11,7 @@ requires "nim >= 1.2.0",
          "nimcrypto >= 0.4.1",
          "https://github.com/ba0f3/dnsclient.nim == 0.1.0",
          "bearssl >= 0.1.4",
-         "chronicles#ba2817f1",
+         "chronicles >= 0.10.2",
          "chronos >= 3.0.6",
          "metrics",
          "secp256k1",
@@ -94,3 +94,34 @@ task examples_build, "Build the samples":
   buildSample("helloworld", true)
   buildTutorial("examples/tutorial_1_connect.md")
   buildTutorial("examples/tutorial_2_customproto.md")
+
+# pin system
+# while nimble lockfile
+# isn't available
+
+const PinFile = ".pinned"
+task pin, "Create a lockfile":
+  # pinner.nim was originally here
+  # but you can't read output from
+  # a command in a nimscript
+  exec "nim c -r tools/pinner.nim"
+
+import sequtils
+import os
+task install_pinned, "Reads the lockfile":
+  let toInstall = readFile(PinFile).splitWhitespace().mapIt((it.split(";", 1)[0], it.split(";", 1)[1]))
+  # [('packageName', 'packageFullUri')]
+
+  rmDir("nimbledeps")
+  mkDir("nimbledeps")
+  exec "nimble install -y " & toInstall.mapIt(it[1]).join(" ")
+
+  # Remove the automatically installed deps
+  # (inefficient you say?)
+  let allowedDirectories = toInstall.mapIt(it[0] & "-" & it[1].split('@')[1])
+  for dependency in listDirs("nimbledeps/pkgs"):
+    if dependency.extractFilename notin allowedDirectories:
+      rmDir(dependency)
+
+task unpin, "Restore global package use":
+  rmDir("nimbledeps")

tools/pinner.nim

@@ -0,0 +1,38 @@
+import os
+import strscans
+import algorithm
+import tables
+import sequtils
+import strutils
+import json
+import osproc
+
+const PinFile = ".pinned"
+removeDir("nimbledeps")
+createDir("nimbledeps")
+discard execCmd("nimble install -dy")
+
+var allDeps: Table[string, string]
+for (_, dependency) in walkDir("nimbledeps/pkgs"):
+  let fileContent = parseJson(readFile(dependency & "/nimblemeta.json"))
+  let url = fileContent.getOrDefault("url").getStr("")
+  var version = fileContent.getOrDefault("vcsRevision").getStr("")
+  var packageName = dependency.split('/')[^1].split('-')[0]
+
+  if version.len == 0:
+    version = execCmdEx("git ls-remote " & url).output.split()[0]
+
+  if version.len > 0 and url.len > 0:
+    let fullPackage = url & "@#" & version
+    if packageName in allDeps and allDeps[packageName] != fullPackage:
+      echo "Warning: duplicate package " & packageName & ":"
+      echo allDeps[packageName]
+      echo fullPackage
+      echo ""
+    allDeps[packageName] = fullPackage
+  else:
+    echo "Failed to get url & version for ", dependency
+
+let asString = toSeq(allDeps.pairs).mapIt(it[0] & ";" & it[1]).sorted().join("\n")
+writeFile(PinFile, asString)
+echo asString