Add utp test to ci (#952)

.github/workflows/fluffy.yml

@@ -8,6 +8,72 @@ on:
       'Makefile', 'nimbus.nimble', '!fluffy/**/*.md']
 
 jobs:
+  # separate job so it can run concurrently with other tests
+  testutp:
+    # whole test setup runs on ubuntu so we do not need multiple arch setup here
+    runs-on: ubuntu-latest
+    # TODO: for now only push event as this way it is easier to get branch name
+    # to build container
+    if: github.event_name == 'push'
+    steps:
+      - name: Checkout nimbus-eth1
+        uses: actions/checkout@v2
+
+      - name: Install modprobe
+        run: |
+          sudo apt-get install -y kmod
+
+      # It is required to correctly run the simulation
+      - name: Load iptables6 kernel modules
+        run: |
+          sudo modprobe ip6table_filter
+
+      - name: Get latest nimbus-build-system commit hash
+        id: versions
+        run: |
+          getHash() {
+            git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1
+          }
+          nbsHash=$(getHash status-im/nimbus-build-system)
+          echo "::set-output name=nimbus_build_system::$nbsHash"
+
+      - name: Restore prebuilt Nim binaries from cache
+        id: nim-cache
+        uses: actions/cache@v2
+        with:
+          path: NimBinaries
+          key: 'nim-linux-amd64-${{ steps.versions.outputs.nimbus_build_system }}'
+
+      - name: Build Nim and Nimbus-eth1 dependencies
+        run: |
+          make -j${ncpu} ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
+
+      - name: build uTP test app container
+        run: |
+          docker build -t test-utp --no-cache --build-arg BRANCH_NAME=${{ github.ref_name }} fluffy/tools/utp_testing/docker
+      
+      - name: run test app with simulator
+        run: |
+          SCENARIO="drop-rate --delay=15ms --bandwidth=10Mbps --queue=25 --rate_to_client=10 --rate_to_server=10" docker-compose -f fluffy/tools/utp_testing/docker/docker-compose.yml up -d
+      
+      - name: wait 5 seconds for containers to start
+        run: |
+          sleep 5
+
+      - name: check containers
+        run: |
+          docker ps -a
+          
+      - name: run uTP test
+        run: |
+          export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
+          DEFAULT_MAKE_FLAGS="-j${ncpu}"
+          env CC=gcc CXX=g++ make ${DEFAULT_MAKE_FLAGS} utp-test
+
+      - name: Stop containers
+        if: always()
+        run: docker-compose -f fluffy/tools/utp_testing/docker/docker-compose.yml down
+
   build:
     strategy:
       fail-fast: false

fluffy/tools/utp_testing/utp_test.nim

@@ -27,7 +27,7 @@ proc generateByteSeqHex(rng: var BrHmacDrbgContext, length: int): string =
 # ./utp_test_app --udp-listen-address=127.0.0.1 --rpc-listen-address=0.0.0.0 --udp-port=9042 --rpc-port=9042
 # or 
 # 1. running in docker dir: docker build -t test-utp --no-cache --build-arg BRANCH_NAME=branch-name .
-# 2. running in docke dir: SCENARIO="scenario name and params " docker-compose up
+# 2. running in docker dir: SCENARIO="scenario name and params " docker-compose up
 procSuite "Utp integration tests":
   let rng = newRng()
   let clientContainerAddress = "127.0.0.1"
@@ -40,13 +40,21 @@ procSuite "Utp integration tests":
 
   # combinator which repeatadly calls passed closure until returned future is 
   # successfull
-  proc repeatTillSuccess[A](f: FutureCallback[A]): Future[A] {.async.}=
+  # TODO: currently works only for non void types
+  proc repeatTillSuccess[A](f: FutureCallback[A], maxTries: int = 10): Future[A] {.async.} =
+    var i = 0
     while true:
       try:
         let res = await f()
         return res
-      except CatchableError:
+      except CatchableError as exc:
+        echo "Call failed due to " & exc.msg
+        inc i
+
+        if i < maxTries:
           continue
+        else:
+          raise exc
       except CancelledError as canc:
         raise canc
 
@@ -70,8 +78,10 @@ procSuite "Utp integration tests":
     await client.connect(clientContainerAddress, clientContainerPort, false)
     await server.connect(serverContainerAddress, serverContainerPort, false)
 
-    let clientInfo = await client.discv5_nodeInfo()
+    # we may need to retry few times if the simm is not ready yet
-    let serverInfo = await server.discv5_nodeInfo()
+    let clientInfo = await repeatTillSuccess(() => client.discv5_nodeInfo(), 10)
+
+    let serverInfo = await repeatTillSuccess(() => server.discv5_nodeInfo(), 10)
 
     # nodes need to have established session before the utp try
     discard await repeatTillSuccess(() => client.discv5_ping(serverInfo.nodeEnr))