From 5484cb707968a03beedf754d3b1c59d634c66c9a Mon Sep 17 00:00:00 2001 From: gusto Date: Mon, 12 Dec 2022 21:29:23 +0200 Subject: [PATCH] ci: add Jenkins and Docker file for PR and doc builds (#23) * ci: add Jenkins and Docker file for PR and doc builds * Fixing waku-bindings dependacy definitions * Add post cleanup and musl-dev to Jenkinsfile * Add go to the Dockerfile for ci * GOCACHE set to tmp for ci builds * Use slim-bullseye docker image for ci builds * Update ci/Jenkinsfile.docs Co-authored-by: Jakub * Add image version and env vars for docs build * Remove duplicated environment section * Add missing rust dependencies * Split jenkins file into linux and macos targets * Removing github actions for PR checks * Add explicit versions to the shell.nix dependencies * Add jenkins libs required for nix * Use default rust-bin version that includes clippy and fmt * Add readme to ci folder Co-authored-by: Jakub --- .github/workflows/pr.yml | 85 ------------------------------- ci/Dockerfile | 22 ++++++++ ci/Jenkinsfile.docs | 40 +++++++++++++++ ci/Jenkinsfile.prs.linux | 46 +++++++++++++++++ ci/Jenkinsfile.prs.macos | 45 ++++++++++++++++ ci/README.md | 26 ++++++++++ nomos-services/network/Cargo.toml | 2 +- shell.nix | 23 +++++++++ 8 files changed, 203 insertions(+), 86 deletions(-) delete mode 100644 .github/workflows/pr.yml create mode 100644 ci/Dockerfile create mode 100644 ci/Jenkinsfile.docs create mode 100644 ci/Jenkinsfile.prs.linux create mode 100644 ci/Jenkinsfile.prs.macos create mode 100644 ci/README.md create mode 100644 shell.nix diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml deleted file mode 100644 index 8e4a4efa..00000000 --- a/.github/workflows/pr.yml +++ /dev/null @@ -1,85 +0,0 @@ -# copy of https://github.com/actions-rs/meta/blob/master/recipes/quickstart.md -# Steps for checking PRs. -on: - pull_request: - branches: - - "*" - -name: PR check - -jobs: - check: - name: Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - uses: actions-rs/cargo@v1 - continue-on-error: true - with: - command: check - - test: - name: Test Suite - if: ${{ !startsWith(github.event.pull_request.title, '[WIP]') && !contains(github.event.label.name, 'DO NOT MERGE') }} - strategy: - fail-fast: false # all OSes should be tested even if one fails (default: true) - matrix: - os: [ubuntu-latest, windows-latest] # macos-latest removed because 1 min on macos is x10 on GH runtime. - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - uses: actions/setup-go@v3 # we need go to build go-waku - with: - go-version: '1.19' - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - uses: actions-rs/cargo@v1 - continue-on-error: true - with: - command: build - args: --all-features - - uses: actions-rs/cargo@v1 - continue-on-error: true - with: - command: test - args: --all-features - - lints: - name: Rust lints - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - components: rustfmt, clippy - - - name: Run cargo fmt - uses: actions-rs/cargo@v1 - continue-on-error: true - with: - command: fmt - args: --all -- --check - - - name: Run cargo clippy - uses: actions-rs/cargo@v1 - continue-on-error: true - with: - command: clippy - args: -- --deny warnings diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 00000000..da794c80 --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,22 @@ +FROM rust:1.65.0-slim-bullseye + +LABEL maintainer="augustinas@status.im" +LABEL source="https://github.com/logos-co/nomos-research" +LABEL description="nomos-research ci build image" + +# Using backports for go 1.19 +RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list + +# Dependecies for publishing documentation and building waku-bindings. +RUN apt-get update && apt-get install -yq \ + openssh-client git python3-pip clang \ + golang-src/bullseye-backports \ + golang-doc/bullseye-backports \ + golang/bullseye-backports + +RUN pip install ghp-import +RUN rustup component add rustfmt clippy + +# Jenkins user needs a specific UID/GID to work. +RUN groupadd -g 1001 jenkins \ + && useradd -u 1001 -g jenkins jenkins diff --git a/ci/Jenkinsfile.docs b/ci/Jenkinsfile.docs new file mode 100644 index 00000000..ef22ac90 --- /dev/null +++ b/ci/Jenkinsfile.docs @@ -0,0 +1,40 @@ +pipeline { + agent { + dockerfile { + label 'linux' + dir 'ci' + } + } + + options { + disableConcurrentBuilds() + buildDiscarder(logRotator( + numToKeepStr: '20', + daysToKeepStr: '30', + )) + } + + environment { + GIT_COMMITTER_NAME = 'status-im-auto' + GIT_COMMITTER_EMAIL = 'auto@status.im' + GIT_SSH_COMMAND = 'ssh -o StrictHostKeyChecking=no' + GOPATH = '/tmp/go' + GOCACHE = '/tmp/' + } + + stages { + stage('Build') { + steps { + sh 'cargo doc --no-deps' + } + } + + stage('Publish') { + steps { + sshagent(credentials: ['status-im-auto-ssh']) { + sh 'ghp-import -p target/doc' + } + } + } + } +} diff --git a/ci/Jenkinsfile.prs.linux b/ci/Jenkinsfile.prs.linux new file mode 100644 index 00000000..0532abdb --- /dev/null +++ b/ci/Jenkinsfile.prs.linux @@ -0,0 +1,46 @@ +pipeline { + agent { + dockerfile { + label 'linux' + dir 'ci' + } + } + + environment { + GOPATH = '/tmp/go' + GOCACHE = '/tmp/' + } + + options { + disableConcurrentBuilds() + buildDiscarder(logRotator( + numToKeepStr: '20', + daysToKeepStr: '30', + )) + } + + stages { + stage('Check') { + steps { + sh 'cargo check' + sh 'cargo fmt -- --check' + sh 'cargo clippy' + } + } + + stage('Build') { + steps { + sh 'cargo build --all --all-features' + } + } + + stage('Test') { + steps { + sh 'cargo test --all --all-features' + } + } + } + post { + cleanup { cleanWs() } + } +} diff --git a/ci/Jenkinsfile.prs.macos b/ci/Jenkinsfile.prs.macos new file mode 100644 index 00000000..5a256de1 --- /dev/null +++ b/ci/Jenkinsfile.prs.macos @@ -0,0 +1,45 @@ +library 'status-jenkins-lib@v1.6.0' + +pipeline { + agent { + label 'macos && x86_64' + } + + environment { + GOPATH = '/tmp/go' + GOCACHE = '/tmp/' + } + + options { + disableConcurrentBuilds() + buildDiscarder(logRotator( + numToKeepStr: '20', + daysToKeepStr: '30', + )) + } + + stages { + stage('Check') { + steps { script { + nix.shell('cargo check') + nix.shell('cargo fmt -- --check') + nix.shell('cargo clippy') + } } + } + + stage('Build') { + steps { script { + nix.shell('cargo build --all --all-features') + } } + } + + stage('Test') { + steps { script { + nix.shell('cargo test --all --all-features') + } } + } + } + post { + cleanup { cleanWs() } + } +} diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..be4d6226 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,26 @@ +# Building `nomos-research` with Jenkins + +This is a short introduction for developers on how to use `ci` folder to update build dependencies or to modify the build process. + +## ci/Dockerfile (Docs, linux target) + +Dockerfile is used when `nomos-research` documentation is being built and to lint/test/build for linux target. Official rust image is used with a predefined version. In addition, golang and cargo components are downloaded when the image is being built. `ghp-import` dependency is added for pushing documentation back to the github repository in the `gh-pages` branch. +In general, this file should be used just for defining dependencies. Related steps and build commands for docs and linux targets should be defined in `ci/Jenkinsfile.prs.linux`. + +## ci/Jenkinsfile.prs.docs + +This file contains the configuration required for Jenkins github user to be able to push the docs to the `gh-pages` branch. It also defines steps for building the documentation and publishing it. + +## ci/Jenkinsfile.prs.linux + +Two most important places in this file are `environment` and `stages`. +* `environment` - variables defined here will be accessible to every stage that runs on an image built from the `ci/Dockerfile` +* `stages` - used to group shell commands that are related to different steps and their layout reflects in the build job summary. + +## ci/Jenkinsfile.prs.macos + +Same as in `Jenkinsfile.prs.macos` the only difference is that instead of Docker image, macos is using `shell.nix` to build a shell with all dependencies. The steps defined here should be identical or similar to what's defined in linux file, just instead of running those commands straight in `sh`, use `nix.shell('command')` wrapper. + +## shell.nix + +Configuration file for the Nix package manager. It defines the build dependencies for `macos` target and can be used to manage and update the dependencies similarly to Dockerfile. diff --git a/nomos-services/network/Cargo.toml b/nomos-services/network/Cargo.toml index 2a72213c..d1f361ba 100644 --- a/nomos-services/network/Cargo.toml +++ b/nomos-services/network/Cargo.toml @@ -22,4 +22,4 @@ tracing-gelf = "0.7" futures = "0.3" [features] -default = ["waku-bindings"] \ No newline at end of file +default = ["waku-bindings"] diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000..dca6d937 --- /dev/null +++ b/shell.nix @@ -0,0 +1,23 @@ +{ pkgs ? import { + builtins = [(import (fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/3389f23412877913b9d22a58dfb241684653d7e9.tar.gz"; + sha256 = "sha256:0wgm7sk9fca38a50hrsqwz6q79z35gqgb9nw80xz7pfdr4jy9pf8"; + }))]; + overlays = [ + (import (fetchGit { + url = "https://github.com/oxalica/rust-overlay.git"; + rev = "fe185fac76e009b4bd543704a8c61077cf70155b"; + })) + ]; + } +}: + +pkgs.mkShell { + name = "nomos-research-build-shell"; + + buildInputs = with pkgs; [ + pkg-config + rust-bin.stable."1.65.0".default + go_1_19 + ]; +}