ci: add Jenkins and Docker file for PR and doc builds (#23)

* ci: add Jenkins and Docker file for PR and doc builds

* Fixing waku-bindings dependacy definitions

* Add post cleanup and musl-dev to Jenkinsfile

* Add go to the Dockerfile for ci

* GOCACHE set to tmp for ci builds

* Use slim-bullseye docker image for ci builds

* Update ci/Jenkinsfile.docs

Co-authored-by: Jakub <i+github@always.fail>

* Add image version and env vars for docs build

* Remove duplicated environment section

* Add missing rust dependencies

* Split jenkins file into linux and macos targets

* Removing github actions for PR checks

* Add explicit versions to the shell.nix dependencies

* Add jenkins libs required for nix

* Use default rust-bin version that includes clippy and fmt

* Add readme to ci folder

Co-authored-by: Jakub <i+github@always.fail>

.github/workflows/pr.yml

@@ -1,85 +0,0 @@
-# copy of https://github.com/actions-rs/meta/blob/master/recipes/quickstart.md
-# Steps for checking PRs.
-on:
-  pull_request:
-    branches:
-      - "*"
-
-name: PR check
-
-jobs:
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        continue-on-error: true
-        with:
-          command: check
-
-  test:
-    name: Test Suite
-    if: ${{ !startsWith(github.event.pull_request.title, '[WIP]') && !contains(github.event.label.name,  'DO NOT MERGE') }}
-    strategy:
-      fail-fast: false # all OSes should be tested even if one fails (default: true)
-      matrix:
-        os: [ubuntu-latest, windows-latest] # macos-latest removed because 1 min on macos is x10 on GH runtime.
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - uses: actions/setup-go@v3 # we need go to build go-waku
-        with:
-          go-version: '1.19'
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        continue-on-error: true
-        with:
-          command: build
-          args: --all-features
-      - uses: actions-rs/cargo@v1
-        continue-on-error: true
-        with:
-          command: test
-          args: --all-features
-
-  lints:
-    name: Rust lints
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
-
-      - name: Run cargo fmt
-        uses: actions-rs/cargo@v1
-        continue-on-error: true
-        with:
-          command: fmt
-          args: --all -- --check
-
-      - name: Run cargo clippy
-        uses: actions-rs/cargo@v1
-        continue-on-error: true
-        with:
-          command: clippy
-          args: -- --deny warnings

ci/Dockerfile

@@ -0,0 +1,22 @@
+FROM rust:1.65.0-slim-bullseye
+
+LABEL maintainer="augustinas@status.im"
+LABEL source="https://github.com/logos-co/nomos-research"
+LABEL description="nomos-research ci build image"
+
+# Using backports for go 1.19
+RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list
+
+# Dependecies for publishing documentation and building waku-bindings.
+RUN apt-get update && apt-get install -yq \
+    openssh-client git python3-pip clang \
+    golang-src/bullseye-backports \
+    golang-doc/bullseye-backports \
+    golang/bullseye-backports
+
+RUN pip install ghp-import
+RUN rustup component add rustfmt clippy
+
+# Jenkins user needs a specific UID/GID to work.
+RUN groupadd -g 1001 jenkins \
+ && useradd -u 1001 -g jenkins jenkins

ci/Jenkinsfile.docs

@@ -0,0 +1,40 @@
+pipeline {
+  agent {
+    dockerfile {
+      label 'linux'
+      dir 'ci'
+    }
+  }
+
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(
+      numToKeepStr: '20',
+      daysToKeepStr: '30',
+    ))
+  }
+
+  environment {
+    GIT_COMMITTER_NAME = 'status-im-auto'
+    GIT_COMMITTER_EMAIL = 'auto@status.im'
+    GIT_SSH_COMMAND = 'ssh -o StrictHostKeyChecking=no'
+    GOPATH  = '/tmp/go'
+    GOCACHE = '/tmp/'
+  }
+
+  stages {
+    stage('Build') {
+      steps {
+        sh 'cargo doc --no-deps'
+      }
+    }
+
+    stage('Publish') {
+      steps {
+        sshagent(credentials: ['status-im-auto-ssh']) {
+          sh 'ghp-import -p target/doc'
+        }
+      }
+    }
+  }
+}

ci/Jenkinsfile.prs.linux

@@ -0,0 +1,46 @@
+pipeline {
+  agent {
+    dockerfile {
+      label 'linux'
+      dir 'ci'
+    }
+  }
+
+  environment {
+    GOPATH  = '/tmp/go'
+    GOCACHE = '/tmp/'
+  }
+
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(
+      numToKeepStr: '20',
+      daysToKeepStr: '30',
+    ))
+  }
+
+  stages {
+    stage('Check') {
+      steps {
+      	sh 'cargo check'
+      	sh 'cargo fmt -- --check'
+	sh 'cargo clippy'
+      }
+    }
+
+    stage('Build') {
+      steps {
+        sh 'cargo build --all --all-features'
+      }
+    }
+
+    stage('Test') {
+      steps {
+        sh 'cargo test --all --all-features'
+      }
+    }
+  }
+  post {
+    cleanup { cleanWs() }
+  }
+}

ci/Jenkinsfile.prs.macos

@@ -0,0 +1,45 @@
+library 'status-jenkins-lib@v1.6.0'
+
+pipeline {
+  agent {
+    label 'macos && x86_64'
+  }
+
+  environment {
+    GOPATH  = '/tmp/go'
+    GOCACHE = '/tmp/'
+  }
+
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(
+      numToKeepStr: '20',
+      daysToKeepStr: '30',
+    ))
+  }
+
+  stages {
+    stage('Check') {
+      steps { script {
+      	nix.shell('cargo check')
+      	nix.shell('cargo fmt -- --check')
+	nix.shell('cargo clippy')
+      } }
+    }
+
+    stage('Build') {
+      steps { script {
+        nix.shell('cargo build --all --all-features')
+      } }
+    }
+
+    stage('Test') {
+      steps { script {
+        nix.shell('cargo test --all --all-features')
+      } }
+    }
+  }
+  post {
+    cleanup { cleanWs() }
+  }
+}

ci/README.md

@@ -0,0 +1,26 @@
+# Building `nomos-research` with Jenkins
+
+This is a short introduction for developers on how to use `ci` folder to update build dependencies or to modify the build process.
+
+## ci/Dockerfile (Docs, linux target)
+
+Dockerfile is used when `nomos-research` documentation is being built and to lint/test/build for linux target. Official rust image is used with a predefined version. In addition, golang and cargo components are downloaded when the image is being built. `ghp-import` dependency is added for pushing documentation back to the github repository in the `gh-pages` branch.
+In general, this file should be used just for defining dependencies. Related steps and build commands for docs and linux targets should be defined in `ci/Jenkinsfile.prs.linux`.
+
+## ci/Jenkinsfile.prs.docs
+
+This file contains the configuration required for Jenkins github user to be able to push the docs to the `gh-pages` branch. It also defines steps for building the documentation and publishing it.
+
+## ci/Jenkinsfile.prs.linux
+
+Two most important places in this file are `environment` and `stages`.
+* `environment` - variables defined here will be accessible to every stage that runs on an image built from the `ci/Dockerfile`
+* `stages` - used to group shell commands that are related to different steps and their layout reflects in the build job summary.
+
+## ci/Jenkinsfile.prs.macos
+
+Same as in `Jenkinsfile.prs.macos` the only difference is that instead of Docker image, macos is using `shell.nix` to build a shell with all dependencies. The steps defined here should be identical or similar to what's defined in linux file, just instead of running those commands straight in `sh`, use `nix.shell('command')` wrapper.
+
+## shell.nix
+
+Configuration file for the Nix package manager. It defines the build dependencies for `macos` target and can be used to manage and update the dependencies similarly to Dockerfile.

nomos-services/network/Cargo.toml

@@ -22,4 +22,4 @@ tracing-gelf = "0.7"
 futures = "0.3"
 
 [features]
-default = ["waku-bindings"]
+default = ["waku-bindings"]

shell.nix

@@ -0,0 +1,23 @@
+{ pkgs ? import <nixpkgs> {
+    builtins = [(import (fetchTarball {
+      url    = "https://github.com/NixOS/nixpkgs/archive/3389f23412877913b9d22a58dfb241684653d7e9.tar.gz";
+      sha256 = "sha256:0wgm7sk9fca38a50hrsqwz6q79z35gqgb9nw80xz7pfdr4jy9pf8";
+    }))];
+    overlays = [
+      (import (fetchGit {
+        url = "https://github.com/oxalica/rust-overlay.git";
+	rev = "fe185fac76e009b4bd543704a8c61077cf70155b";
+      }))
+    ];
+   }
+}:
+
+pkgs.mkShell {
+  name = "nomos-research-build-shell";
+
+  buildInputs = with pkgs; [
+    pkg-config
+    rust-bin.stable."1.65.0".default
+    go_1_19
+  ];
+}