From 792b73b358ef4f1c759964159a7214e728ef3882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?rich=CE=9Brd?= Date: Wed, 22 Nov 2023 09:13:23 -0400 Subject: [PATCH] chore: extract linux build to separate workflow (#918) --- .github/workflows/build_linux_pkgs.yml | 78 +++++++++++++++ .github/workflows/build_mobile.yml | 6 -- .github/workflows/ci.yml | 91 +++++------------- ci/Jenkinsfile | 99 ------------------- ci/Jenkinsfile.android | 82 ---------------- ci/Jenkinsfile.ios | 78 --------------- ci/Jenkinsfile.linux | 89 ----------------- ci/Jenkinsfile.tests | 126 ------------------------- 8 files changed, 99 insertions(+), 550 deletions(-) create mode 100644 .github/workflows/build_linux_pkgs.yml delete mode 100644 ci/Jenkinsfile delete mode 100644 ci/Jenkinsfile.android delete mode 100644 ci/Jenkinsfile.ios delete mode 100644 ci/Jenkinsfile.linux delete mode 100644 ci/Jenkinsfile.tests diff --git a/.github/workflows/build_linux_pkgs.yml b/.github/workflows/build_linux_pkgs.yml new file mode 100644 index 00000000..59b6d2df --- /dev/null +++ b/.github/workflows/build_linux_pkgs.yml @@ -0,0 +1,78 @@ +name: build_linux_pkgs + +on: + push: + tags: + - 'v*' # "e.g. v0.4" + + workflow_dispatch: + +jobs: + env: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: xom9ikk/dotenv@v2 + with: + path: ".github/" + - run: | + echo "go_version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT + + - run: | + VERSION=$(cat ./VERSION) + echo "waku_version=$VERSION" >> $GITHUB_OUTPUT + + + build-linux: + needs: env + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + matrix: + ext: [deb, rpm] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Get submodules hash + id: submodules + run: | + echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT + + - name: Cache submodules + uses: actions/cache@v3 + with: + path: | + vendor/ + .git/modules + key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: ${{ needs.env.outputs.go_version }} + cache: false + + - name: Build + id: build + run: | + make build + mkdir ./build/linux + cp ./build/waku ./build/linux/. + strip --strip-unneeded ./build/linux/waku + + - name: Package ${{ matrix.ext }} + uses: bpicode/github-action-fpm@master + with: + fpm_args: ./build/linux/waku=/usr/bin/waku + fpm_opts: '-p gowaku-${{ needs.env.outputs.waku_version }}-x86_64.${{ matrix.ext }} -n go-waku -t ${{ matrix.ext }} -s dir --license "MIT, Apache 2.0" --version ${{ needs.env.outputs.waku_version }} --architecture x86_64 --depends libc6 --description "Go implementation of Waku v2 protocols" --url "https://github.com/waku-org/go-waku" --maintainer "Richard Ramos "' + + - name: Upload ${{ matrix.ext }} + uses: actions/upload-artifact@v3 + with: + name: gowaku-${{ needs.env.outputs.waku_version }}-x86_64.${{ matrix.ext }} + path: ./gowaku-${{ needs.env.outputs.waku_version }}-x86_64.${{ matrix.ext }} + if-no-files-found: error diff --git a/.github/workflows/build_mobile.yml b/.github/workflows/build_mobile.yml index 5b39d8c2..2ae22dba 100644 --- a/.github/workflows/build_mobile.yml +++ b/.github/workflows/build_mobile.yml @@ -88,12 +88,6 @@ jobs: .git/modules key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} - - name: Read VERSION file - id: waku_version - run: | - VERSION=$(cat ./VERSION) - echo "version=$VERSION" >> $GITHUB_OUTPUT - - name: Install Go uses: actions/setup-go@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92aa0f54..d055658a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,9 +6,6 @@ on: branches: - master -env: - GO_VERSION: '1.20' - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -46,9 +43,24 @@ jobs: v2: ${{ steps.filter.outputs.v2 }} docker: ${{ steps.filter.outputs.docker }} + env: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: xom9ikk/dotenv@v2 + with: + path: ".github/" + - run: | + echo "go_version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT + + - run: | + VERSION=$(cat ./VERSION) + echo "waku_version=$VERSION" >> $GITHUB_OUTPUT + golangci: name: lint - needs: changes + needs: [changes, env] if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' }} runs-on: ubuntu-latest steps: @@ -58,7 +70,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: ${{ env.GO_VERSION }} + go-version: ${{ needs.env.outputs.go_version }} cache: false - name: Execute golangci-lint @@ -68,7 +80,7 @@ jobs: args: --deadline=5m build: - needs: changes + needs: [changes, env] if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' }} strategy: matrix: @@ -97,7 +109,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: ${{ env.GO_VERSION }} + go-version: ${{ needs.env.outputs.go_version }} cache: false - name: Build binary @@ -109,68 +121,8 @@ jobs: - name: Build examples run: make build-example - build-linux: - needs: changes - if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' }} - runs-on: ubuntu-latest - timeout-minutes: 60 - - strategy: - matrix: - ext: [deb, rpm] - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Get submodules hash - id: submodules - run: | - echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT - - - name: Cache submodules - uses: actions/cache@v3 - with: - path: | - vendor/ - .git/modules - key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} - - - name: Read VERSION file - id: waku_version - run: | - VERSION=$(cat ./VERSION) - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - - - name: Build - id: build - run: | - make build - mkdir ./build/linux - cp ./build/waku ./build/linux/. - strip --strip-unneeded ./build/linux/waku - - - name: Package ${{ matrix.ext }} - uses: bpicode/github-action-fpm@master - with: - fpm_args: ./build/linux/waku=/usr/bin/waku - fpm_opts: '-p gowaku-${{ steps.waku_version.outputs.version }}-x86_64.${{ matrix.ext }} -n go-waku -t ${{ matrix.ext }} -s dir --license "MIT, Apache 2.0" --version ${{ steps.waku_version.outputs.version }} --architecture x86_64 --depends libc6 --description "Go implementation of Waku v2 protocols" --url "https://github.com/waku-org/go-waku" --maintainer "Richard Ramos "' - - - name: Upload ${{ matrix.ext }} - uses: actions/upload-artifact@v3 - with: - name: gowaku-${{ steps.waku_version.outputs.version }}-x86_64.${{ matrix.ext }} - path: ./gowaku-${{ steps.waku_version.outputs.version }}-x86_64.${{ matrix.ext }} - if-no-files-found: error - test: - needs: changes + needs: [changes, env] if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' }} strategy: matrix: @@ -199,7 +151,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: ${{ env.GO_VERSION }} + go-version: ${{ needs.env.outputs.go_version }} cache: false - name: "Run tests" @@ -214,4 +166,3 @@ jobs: run: | docker compose -f .github/docker-compose/postgres.yml up -d make test-postgres${{ matrix.tests == 'test-with-race' && '-with-race' || '' }} - diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile deleted file mode 100644 index f772c292..00000000 --- a/ci/Jenkinsfile +++ /dev/null @@ -1,99 +0,0 @@ -library 'status-jenkins-lib@v1.7.0' - -pipeline { - agent { label 'linux' } - - options { - timestamps() - disableConcurrentBuilds() - /* Prevent Jenkins jobs from running forever */ - timeout(time: 40, unit: 'MINUTES') - /* Limit builds retained */ - buildDiscarder(logRotator( - numToKeepStr: '10', - daysToKeepStr: '30', - artifactNumToKeepStr: '10', - )) - } - - /* WARNING: Defining parameters here with the ?: trick causes them to remember last value. */ - parameters { - booleanParam( - name: 'PUBLISH', - description: 'Trigger publishing of build results for nightly or release.', - defaultValue: getPublishDefault(params.PUBLISH), - ) - } - - stages { - stage('Build') { - parallel { - stage('iOS') { steps { script { - ios = jenkins.Build('go-waku/platforms/ios') - } } } - stage('Android') { steps { script { - android = jenkins.Build('go-waku/platforms/android') - } } } - stage('Linux') { steps { script { - linux = jenkins.Build('go-waku/platforms/linux') - } } } - } - } - stage('Archive') { - steps { script { - sh('rm -f pkg/*') - jenkins.copyArts(ios) - jenkins.copyArts(android) - jenkins.copyArts(linux) - sha = "pkg/${utils.pkgFilename(ext: 'sha256')}" - dir('pkg') { - /* generate sha256 checksums for upload */ - sh "sha256sum * | tee ../${sha}" - archiveArtifacts('*') - } - } } - } - stage('Upload') { - steps { script { - /* object for easier URLs handling */ - urls = [ - /* mobile */ - Android: utils.pkgUrl(android), - iOS: utils.pkgUrl(ios), - Linux: utils.pkgUrl(linux), - /* upload the sha256 checksums file too */ - SHA: s3.uploadArtifact(sha), - ] - /* add URLs to the build description */ - jenkins.setBuildDesc(urls) - } } - } - stage('Publish') { - when { expression { params.PUBLISH } } - steps { script { - github.publishReleaseFiles(repo: 'status-desktop'); - } } - } - } -} - -/* Helper that generates list of available choices for a parameter - * but re-orders them based on the currently set value. First is default. */ -def List genChoices(String previousChoice, List defaultChoices) { - if (previousChoice == null) { - return defaultChoices - } - choices = defaultChoices.minus(previousChoice) - choices.add(0, previousChoice) - return choices -} - -/* Helper that makes PUBLISH default to 'false' unless: - * - The build is for a release branch - * - A user explicitly specified a value - * Since release builds create and re-create GitHub drafts every time. */ -def Boolean getPublishDefault(Boolean previousValue) { - if (env.JOB_NAME.startsWith('go-waku/release')) { return true } - if (previousValue != null) { return previousValue } - return false -} diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android deleted file mode 100644 index acfb6901..00000000 --- a/ci/Jenkinsfile.android +++ /dev/null @@ -1,82 +0,0 @@ -library 'status-jenkins-lib@v1.7.0' - -pipeline { - agent { label 'linux && nix-2.11 && x86_64' } - - options { - timestamps() - disableConcurrentBuilds() - /* Prevent Jenkins jobs from running forever */ - timeout(time: 30, unit: 'MINUTES') - /* Go requires a certain directory structure */ - checkoutToSubdirectory('src/github.com/waku-org/go-waku') - /* Limit builds retained */ - buildDiscarder(logRotator( - numToKeepStr: '10', - daysToKeepStr: '20', - artifactNumToKeepStr: '10', - )) - /* Allows combined build to copy */ - copyArtifactPermission('/go-waku/*') - } - - environment { - CC = "gcc-10" - /* Other stuff */ - TARGET = 'android' - REPO = "${env.WORKSPACE}/src/github.com/waku-org/go-waku" - GOCACHE = "${env.WORKSPACE_TMP}/go-build" - GOPATH = "${env.WORKSPACE}" - PATH = "${env.PATH}:${env.GOPATH}/bin" - /* Android SDK */ - ANDROID_HOME = '/usr/lib/android-sdk' - ANDROID_SDK_ROOT = '/usr/lib/android-sdk' - /* gomobile requires a specific NDK version */ - ANDROID_NDK = "/opt/android-ndk-r23c" - ANDROID_NDK_HOME = "/opt/android-ndk-r23c" - } - - stages { - - stage('Prep') { steps { script { dir(env.REPO) { - env.ARTIFACT = "${env.REPO}/pkg/" + utils.pkgFilename( - name: "go-waku", - type: "android", - ext: "tar.gz" - ) - - nix.develop('make install-gomobile', pure: false) - } } } } - - stage('Build') { steps { script { dir(env.REPO) { - /* First gomobile run always fails. - * https://github.com/golang/go/issues/37372 */ - nix.develop('make mobile-android || make mobile-android', pure: false) - dir('build/lib') { - sh 'tar -czvf gowaku-android.tar.gz gowaku.aar gowaku-sources.jar' - sh "cp gowaku-android.tar.gz ${env.ARTIFACT}" - } - } } } } - - stage('Parallel Upload') { - parallel { - stage('Archive') { - steps { script { - archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/")) - } } - } - stage('Upload') { - steps { script { - env.PKG_URL = s3.uploadArtifact(env.ARTIFACT) - jenkins.setBuildDesc(android: env.PKG_URL) - } } - } - } - } - } - post { - success { script { github.notifyPR(true) } } - failure { script { github.notifyPR(false) } } - always { cleanWs() } - } -} diff --git a/ci/Jenkinsfile.ios b/ci/Jenkinsfile.ios deleted file mode 100644 index e4deba90..00000000 --- a/ci/Jenkinsfile.ios +++ /dev/null @@ -1,78 +0,0 @@ -library 'status-jenkins-lib@v1.7.0' - -pipeline { - agent { label 'macos && nix-2.11 && aarch64' } - - options { - timestamps() - /* Prevent Jenkins jobs from running forever */ - timeout(time: 30, unit: 'MINUTES') - /* Go requires a certain directory structure */ - checkoutToSubdirectory('src/github.com/waku-org/go-waku') - /* Limit builds retained */ - buildDiscarder(logRotator( - numToKeepStr: '10', - daysToKeepStr: '20', - artifactNumToKeepStr: '10', - )) - /* Allows combined build to copy */ - copyArtifactPermission('/go-waku/*') - } - - environment { - TARGET = 'ios' - REPO = "${env.WORKSPACE}/src/github.com/waku-org/go-waku" - GOCACHE = "${env.WORKSPACE_TMP}/go-build" - GOPATH = "${env.WORKSPACE}" - PATH = "${env.PATH}:${env.GOPATH}/bin" - } - - stages { - - stage('Prep') { steps { script { dir(env.REPO) { - env.ARTIFACT = "${env.REPO}/pkg/" + utils.pkgFilename( - name: "go-waku", - type: "ios", - ext: "tar.gz" - ) - sh 'make install-gomobile' - } } } } - - stage('Build') { - steps { script { dir(env.REPO) { - nix.develop('which xcodebuild', pure: false) - nix.develop('make mobile-ios', pure: false) - } } } - } - - stage('Package') { - steps { dir(env.REPO) { - dir('build/lib') { - sh 'tar -czvf gowaku-ios.tar.gz Gowaku.xcframework' - sh "cp gowaku-ios.tar.gz ${env.ARTIFACT}" - } - } } - } - - stage('Parallel Upload') { - parallel { - stage('Archive') { - steps { script { - archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/")) - } } - } - stage('Upload') { - steps { script { - env.PKG_URL = s3.uploadArtifact(env.ARTIFACT) - jenkins.setBuildDesc(ios: env.PKG_URL) - } } - } - } - } - } - post { - success { script { github.notifyPR(true) } } - failure { script { github.notifyPR(false) } } - always { cleanWs() } - } -} diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux deleted file mode 100644 index 1ad0854d..00000000 --- a/ci/Jenkinsfile.linux +++ /dev/null @@ -1,89 +0,0 @@ -library 'status-jenkins-lib@v1.7.0' - -pipeline { - agent { - label 'linux && nix-2.11 && x86_64' - } - - options { - timestamps() - disableConcurrentBuilds() - /* Prevent Jenkins jobs from running forever */ - timeout(time: 30, unit: 'MINUTES') - /* Go requires a certain directory structure */ - checkoutToSubdirectory('src/github.com/waku-org/go-waku') - /* Limit builds retained */ - buildDiscarder(logRotator( - numToKeepStr: '10', - daysToKeepStr: '20', - artifactNumToKeepStr: '10', - )) - /* Allows combined build to copy */ - copyArtifactPermission('/go-waku/*') - } - - environment { - TARGET = 'linux' - REPO = "${env.WORKSPACE}/src/github.com/waku-org/go-waku" - GOCACHE = "${env.WORKSPACE_TMP}/go-build" - GOPATH = "${env.WORKSPACE}" - PATH = "${env.PATH}:${env.GOPATH}/bin" - } - - stages { - - stage('Prep') { - steps { script { dir(env.REPO) { - env.DEB_ARTIFACT = "${env.REPO}/pkg/" + utils.pkgFilename( - name: "go-waku", - type: "x86_64", - ext: "deb" - ) - } } } - } - - stage('Build') { - steps { script { dir(env.REPO) { - nix.develop('make build') - } } } - } - - stage('Build examples') { - steps { script { dir(env.REPO) { - nix.develop('make build-example') - } } } - } - - stage('Package') { - steps { script { dir(env.REPO) { - dir('./scripts/linux') { - nix.develop('./fpm-build.sh', attr: 'fpm') - } - dir('build') { - sh "cp gowaku*.deb ${env.DEB_ARTIFACT}" - } - } } } - } - - stage('Parallel Upload') { - parallel { - stage('Archive') { - steps { script { - archiveArtifacts(env.DEB_ARTIFACT.minus("${env.WORKSPACE}/")) - } } - } - stage('Upload') { - steps { script { - env.PKG_URL = s3.uploadArtifact(env.DEB_ARTIFACT) - jenkins.setBuildDesc(x86_64_deb: env.PKG_URL) - } } - } - } - } - } - post { - success { script { github.notifyPR(true) } } - failure { script { github.notifyPR(false) } } - always { cleanWs() } - } -} diff --git a/ci/Jenkinsfile.tests b/ci/Jenkinsfile.tests deleted file mode 100644 index 87cb3363..00000000 --- a/ci/Jenkinsfile.tests +++ /dev/null @@ -1,126 +0,0 @@ -library 'status-jenkins-lib@v1.7.0' - -pipeline { - agent { - label 'linux && nix-2.11 && x86_64' - } - - options { - timestamps() - disableConcurrentBuilds() - /* Prevent Jenkins jobs from running forever */ - timeout(time: 30, unit: 'MINUTES') - /* Go requires a certain directory structure */ - checkoutToSubdirectory('src/github.com/waku-org/go-waku') - buildDiscarder(logRotator( - numToKeepStr: '10', - daysToKeepStr: '30', - )) - } - - /* WARNING: Defining parameters here with the ?: trick causes them to remember last value. */ - parameters { - booleanParam( - name: 'RACE', - description: 'Run tests with check for race condition.', - defaultValue: getRaceDefault() - ) - } - - - environment { - TARGET = 'tests' - DB_CONT = "go-waku-test-db-${env.EXECUTOR_NUMBER.toInteger() + 1}" - DB_PORT = "${5432 + env.EXECUTOR_NUMBER.toInteger()}" - REPO = "${env.WORKSPACE}/src/github.com/waku-org/go-waku" - GOCACHE = "${env.WORKSPACE_TMP}/go-build" - GOPATH = "${env.WORKSPACE}/go" - PATH = "${env.PATH}:${env.GOPATH}/bin" - /* Necesary to avoid cache poisoning by other builds. */ - GOLANGCI_LINT_CACHE = "${env.WORKSPACE_TMP}/golangci-lint" - /* Ganache config */ - GANACHE_RPC_PORT = "${8989 + env.EXECUTOR_NUMBER.toInteger()}" - GANACHE_MNEMONIC = 'swim relax risk shy chimney please usual search industry board music segment' - } - - stages { - stage('Lint') { - steps { script { dir(env.REPO) { - nix.develop('make lint', pure: false) - } } } - } - - stage('Test') { - steps { script { dir(env.REPO) { - if (params.RACE) { - nix.develop('make test-with-race', pure: false) - }else { - nix.develop('make test-ci', pure: false) - } - } } } - } - - stage('PostgresSQL') { - environment { - TEST_DB_PORT = "${env.DB_PORT}" - } - steps { script { dir(env.REPO) { - db = docker.image('postgres:9.6-alpine').withRun([ - "--name=${DB_CONT}", - "--env=POSTGRES_HOST_AUTH_METHOD=trust", - "--publish=${DB_PORT}:${DB_PORT}", - ].join(' '), "-p ${DB_PORT}") { c -> - if (params.RACE) { - nix.develop('make test-postgres-with-race', pure: false) - }else { - nix.develop('make test-postgres', pure: false) - } - } - } } } - post { cleanup { /* Leftover DB containers. */ - sh "docker rm ${DB_CONT} || true" - } } - } - - stage('Ganache') { - steps { script { - ganache = docker.image( - 'trufflesuite/ganache:v7.4.1' - ).run( - "-p 127.0.0.1:${env.GANACHE_RPC_PORT}:8545", - "-m='${GANACHE_MNEMONIC}'" - ) - } } - } - - stage('On-chain tests') { - environment { - GANACHE_NETWORK_RPC_URL = "ws://localhost:${env.GANACHE_RPC_PORT}" - } - steps { script { dir(env.REPO) { - if (params.RACE) { - nix.develop('make test-onchain-with-race', pure: false) - }else { - nix.develop('make test-onchain', pure: false) - } - } } } - } - } - post { - always { script { /* No artifact but a PKG_URL is necessary. */ - env.PKG_URL = "${currentBuild.absoluteUrl}consoleText" - } } - success { script { github.notifyPR(true) } } - failure { script { github.notifyPR(false) } } - cleanup { script { - cleanWs() - catchError { - ganache.stop() - } - } } - } -} - -def Boolean getRaceDefault() { - return env.JOB_NAME.split('/').contains('race') -}