chore: extract linux build to separate workflow (#918)

This commit is contained in:
richΛrd 2023-11-22 09:13:23 -04:00 committed by GitHub
parent 2af8cf7344
commit 792b73b358
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 99 additions and 550 deletions

78
.github/workflows/build_linux_pkgs.yml vendored Normal file
View File

@ -0,0 +1,78 @@
name: build_linux_pkgs
on:
push:
tags:
- 'v*' # "e.g. v0.4"
workflow_dispatch:
jobs:
env:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: xom9ikk/dotenv@v2
with:
path: ".github/"
- run: |
echo "go_version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT
- run: |
VERSION=$(cat ./VERSION)
echo "waku_version=$VERSION" >> $GITHUB_OUTPUT
build-linux:
needs: env
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
ext: [deb, rpm]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Get submodules hash
id: submodules
run: |
echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
- name: Cache submodules
uses: actions/cache@v3
with:
path: |
vendor/
.git/modules
key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: ${{ needs.env.outputs.go_version }}
cache: false
- name: Build
id: build
run: |
make build
mkdir ./build/linux
cp ./build/waku ./build/linux/.
strip --strip-unneeded ./build/linux/waku
- name: Package ${{ matrix.ext }}
uses: bpicode/github-action-fpm@master
with:
fpm_args: ./build/linux/waku=/usr/bin/waku
fpm_opts: '-p gowaku-${{ needs.env.outputs.waku_version }}-x86_64.${{ matrix.ext }} -n go-waku -t ${{ matrix.ext }} -s dir --license "MIT, Apache 2.0" --version ${{ needs.env.outputs.waku_version }} --architecture x86_64 --depends libc6 --description "Go implementation of Waku v2 protocols" --url "https://github.com/waku-org/go-waku" --maintainer "Richard Ramos <richard@status.im>"'
- name: Upload ${{ matrix.ext }}
uses: actions/upload-artifact@v3
with:
name: gowaku-${{ needs.env.outputs.waku_version }}-x86_64.${{ matrix.ext }}
path: ./gowaku-${{ needs.env.outputs.waku_version }}-x86_64.${{ matrix.ext }}
if-no-files-found: error

View File

@ -88,12 +88,6 @@ jobs:
.git/modules
key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
- name: Read VERSION file
id: waku_version
run: |
VERSION=$(cat ./VERSION)
echo "version=$VERSION" >> $GITHUB_OUTPUT
- name: Install Go
uses: actions/setup-go@v4
with:

View File

@ -6,9 +6,6 @@ on:
branches:
- master
env:
GO_VERSION: '1.20'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
@ -46,9 +43,24 @@ jobs:
v2: ${{ steps.filter.outputs.v2 }}
docker: ${{ steps.filter.outputs.docker }}
env:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: xom9ikk/dotenv@v2
with:
path: ".github/"
- run: |
echo "go_version=${{ env.GO_VERSION }}" >> $GITHUB_OUTPUT
- run: |
VERSION=$(cat ./VERSION)
echo "waku_version=$VERSION" >> $GITHUB_OUTPUT
golangci:
name: lint
needs: changes
needs: [changes, env]
if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' }}
runs-on: ubuntu-latest
steps:
@ -58,7 +70,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
go-version: ${{ needs.env.outputs.go_version }}
cache: false
- name: Execute golangci-lint
@ -68,7 +80,7 @@ jobs:
args: --deadline=5m
build:
needs: changes
needs: [changes, env]
if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' }}
strategy:
matrix:
@ -97,7 +109,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
go-version: ${{ needs.env.outputs.go_version }}
cache: false
- name: Build binary
@ -109,68 +121,8 @@ jobs:
- name: Build examples
run: make build-example
build-linux:
needs: changes
if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
ext: [deb, rpm]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Get submodules hash
id: submodules
run: |
echo "hash=$(git submodule status | awk '{print $1}' | sort | shasum -a 256 | sed 's/[ -]*//g')" >> $GITHUB_OUTPUT
- name: Cache submodules
uses: actions/cache@v3
with:
path: |
vendor/
.git/modules
key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }}
- name: Read VERSION file
id: waku_version
run: |
VERSION=$(cat ./VERSION)
echo "version=$VERSION" >> $GITHUB_OUTPUT
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
cache: false
- name: Build
id: build
run: |
make build
mkdir ./build/linux
cp ./build/waku ./build/linux/.
strip --strip-unneeded ./build/linux/waku
- name: Package ${{ matrix.ext }}
uses: bpicode/github-action-fpm@master
with:
fpm_args: ./build/linux/waku=/usr/bin/waku
fpm_opts: '-p gowaku-${{ steps.waku_version.outputs.version }}-x86_64.${{ matrix.ext }} -n go-waku -t ${{ matrix.ext }} -s dir --license "MIT, Apache 2.0" --version ${{ steps.waku_version.outputs.version }} --architecture x86_64 --depends libc6 --description "Go implementation of Waku v2 protocols" --url "https://github.com/waku-org/go-waku" --maintainer "Richard Ramos <richard@status.im>"'
- name: Upload ${{ matrix.ext }}
uses: actions/upload-artifact@v3
with:
name: gowaku-${{ steps.waku_version.outputs.version }}-x86_64.${{ matrix.ext }}
path: ./gowaku-${{ steps.waku_version.outputs.version }}-x86_64.${{ matrix.ext }}
if-no-files-found: error
test:
needs: changes
needs: [changes, env]
if: ${{ needs.changes.outputs.v2 == 'true' || needs.changes.outputs.common == 'true' }}
strategy:
matrix:
@ -199,7 +151,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: ${{ env.GO_VERSION }}
go-version: ${{ needs.env.outputs.go_version }}
cache: false
- name: "Run tests"
@ -214,4 +166,3 @@ jobs:
run: |
docker compose -f .github/docker-compose/postgres.yml up -d
make test-postgres${{ matrix.tests == 'test-with-race' && '-with-race' || '' }}

99
ci/Jenkinsfile vendored
View File

@ -1,99 +0,0 @@
library 'status-jenkins-lib@v1.7.0'
pipeline {
agent { label 'linux' }
options {
timestamps()
disableConcurrentBuilds()
/* Prevent Jenkins jobs from running forever */
timeout(time: 40, unit: 'MINUTES')
/* Limit builds retained */
buildDiscarder(logRotator(
numToKeepStr: '10',
daysToKeepStr: '30',
artifactNumToKeepStr: '10',
))
}
/* WARNING: Defining parameters here with the ?: trick causes them to remember last value. */
parameters {
booleanParam(
name: 'PUBLISH',
description: 'Trigger publishing of build results for nightly or release.',
defaultValue: getPublishDefault(params.PUBLISH),
)
}
stages {
stage('Build') {
parallel {
stage('iOS') { steps { script {
ios = jenkins.Build('go-waku/platforms/ios')
} } }
stage('Android') { steps { script {
android = jenkins.Build('go-waku/platforms/android')
} } }
stage('Linux') { steps { script {
linux = jenkins.Build('go-waku/platforms/linux')
} } }
}
}
stage('Archive') {
steps { script {
sh('rm -f pkg/*')
jenkins.copyArts(ios)
jenkins.copyArts(android)
jenkins.copyArts(linux)
sha = "pkg/${utils.pkgFilename(ext: 'sha256')}"
dir('pkg') {
/* generate sha256 checksums for upload */
sh "sha256sum * | tee ../${sha}"
archiveArtifacts('*')
}
} }
}
stage('Upload') {
steps { script {
/* object for easier URLs handling */
urls = [
/* mobile */
Android: utils.pkgUrl(android),
iOS: utils.pkgUrl(ios),
Linux: utils.pkgUrl(linux),
/* upload the sha256 checksums file too */
SHA: s3.uploadArtifact(sha),
]
/* add URLs to the build description */
jenkins.setBuildDesc(urls)
} }
}
stage('Publish') {
when { expression { params.PUBLISH } }
steps { script {
github.publishReleaseFiles(repo: 'status-desktop');
} }
}
}
}
/* Helper that generates list of available choices for a parameter
* but re-orders them based on the currently set value. First is default. */
def List genChoices(String previousChoice, List defaultChoices) {
if (previousChoice == null) {
return defaultChoices
}
choices = defaultChoices.minus(previousChoice)
choices.add(0, previousChoice)
return choices
}
/* Helper that makes PUBLISH default to 'false' unless:
* - The build is for a release branch
* - A user explicitly specified a value
* Since release builds create and re-create GitHub drafts every time. */
def Boolean getPublishDefault(Boolean previousValue) {
if (env.JOB_NAME.startsWith('go-waku/release')) { return true }
if (previousValue != null) { return previousValue }
return false
}

View File

@ -1,82 +0,0 @@
library 'status-jenkins-lib@v1.7.0'
pipeline {
agent { label 'linux && nix-2.11 && x86_64' }
options {
timestamps()
disableConcurrentBuilds()
/* Prevent Jenkins jobs from running forever */
timeout(time: 30, unit: 'MINUTES')
/* Go requires a certain directory structure */
checkoutToSubdirectory('src/github.com/waku-org/go-waku')
/* Limit builds retained */
buildDiscarder(logRotator(
numToKeepStr: '10',
daysToKeepStr: '20',
artifactNumToKeepStr: '10',
))
/* Allows combined build to copy */
copyArtifactPermission('/go-waku/*')
}
environment {
CC = "gcc-10"
/* Other stuff */
TARGET = 'android'
REPO = "${env.WORKSPACE}/src/github.com/waku-org/go-waku"
GOCACHE = "${env.WORKSPACE_TMP}/go-build"
GOPATH = "${env.WORKSPACE}"
PATH = "${env.PATH}:${env.GOPATH}/bin"
/* Android SDK */
ANDROID_HOME = '/usr/lib/android-sdk'
ANDROID_SDK_ROOT = '/usr/lib/android-sdk'
/* gomobile requires a specific NDK version */
ANDROID_NDK = "/opt/android-ndk-r23c"
ANDROID_NDK_HOME = "/opt/android-ndk-r23c"
}
stages {
stage('Prep') { steps { script { dir(env.REPO) {
env.ARTIFACT = "${env.REPO}/pkg/" + utils.pkgFilename(
name: "go-waku",
type: "android",
ext: "tar.gz"
)
nix.develop('make install-gomobile', pure: false)
} } } }
stage('Build') { steps { script { dir(env.REPO) {
/* First gomobile run always fails.
* https://github.com/golang/go/issues/37372 */
nix.develop('make mobile-android || make mobile-android', pure: false)
dir('build/lib') {
sh 'tar -czvf gowaku-android.tar.gz gowaku.aar gowaku-sources.jar'
sh "cp gowaku-android.tar.gz ${env.ARTIFACT}"
}
} } } }
stage('Parallel Upload') {
parallel {
stage('Archive') {
steps { script {
archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/"))
} }
}
stage('Upload') {
steps { script {
env.PKG_URL = s3.uploadArtifact(env.ARTIFACT)
jenkins.setBuildDesc(android: env.PKG_URL)
} }
}
}
}
}
post {
success { script { github.notifyPR(true) } }
failure { script { github.notifyPR(false) } }
always { cleanWs() }
}
}

View File

@ -1,78 +0,0 @@
library 'status-jenkins-lib@v1.7.0'
pipeline {
agent { label 'macos && nix-2.11 && aarch64' }
options {
timestamps()
/* Prevent Jenkins jobs from running forever */
timeout(time: 30, unit: 'MINUTES')
/* Go requires a certain directory structure */
checkoutToSubdirectory('src/github.com/waku-org/go-waku')
/* Limit builds retained */
buildDiscarder(logRotator(
numToKeepStr: '10',
daysToKeepStr: '20',
artifactNumToKeepStr: '10',
))
/* Allows combined build to copy */
copyArtifactPermission('/go-waku/*')
}
environment {
TARGET = 'ios'
REPO = "${env.WORKSPACE}/src/github.com/waku-org/go-waku"
GOCACHE = "${env.WORKSPACE_TMP}/go-build"
GOPATH = "${env.WORKSPACE}"
PATH = "${env.PATH}:${env.GOPATH}/bin"
}
stages {
stage('Prep') { steps { script { dir(env.REPO) {
env.ARTIFACT = "${env.REPO}/pkg/" + utils.pkgFilename(
name: "go-waku",
type: "ios",
ext: "tar.gz"
)
sh 'make install-gomobile'
} } } }
stage('Build') {
steps { script { dir(env.REPO) {
nix.develop('which xcodebuild', pure: false)
nix.develop('make mobile-ios', pure: false)
} } }
}
stage('Package') {
steps { dir(env.REPO) {
dir('build/lib') {
sh 'tar -czvf gowaku-ios.tar.gz Gowaku.xcframework'
sh "cp gowaku-ios.tar.gz ${env.ARTIFACT}"
}
} }
}
stage('Parallel Upload') {
parallel {
stage('Archive') {
steps { script {
archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/"))
} }
}
stage('Upload') {
steps { script {
env.PKG_URL = s3.uploadArtifact(env.ARTIFACT)
jenkins.setBuildDesc(ios: env.PKG_URL)
} }
}
}
}
}
post {
success { script { github.notifyPR(true) } }
failure { script { github.notifyPR(false) } }
always { cleanWs() }
}
}

View File

@ -1,89 +0,0 @@
library 'status-jenkins-lib@v1.7.0'
pipeline {
agent {
label 'linux && nix-2.11 && x86_64'
}
options {
timestamps()
disableConcurrentBuilds()
/* Prevent Jenkins jobs from running forever */
timeout(time: 30, unit: 'MINUTES')
/* Go requires a certain directory structure */
checkoutToSubdirectory('src/github.com/waku-org/go-waku')
/* Limit builds retained */
buildDiscarder(logRotator(
numToKeepStr: '10',
daysToKeepStr: '20',
artifactNumToKeepStr: '10',
))
/* Allows combined build to copy */
copyArtifactPermission('/go-waku/*')
}
environment {
TARGET = 'linux'
REPO = "${env.WORKSPACE}/src/github.com/waku-org/go-waku"
GOCACHE = "${env.WORKSPACE_TMP}/go-build"
GOPATH = "${env.WORKSPACE}"
PATH = "${env.PATH}:${env.GOPATH}/bin"
}
stages {
stage('Prep') {
steps { script { dir(env.REPO) {
env.DEB_ARTIFACT = "${env.REPO}/pkg/" + utils.pkgFilename(
name: "go-waku",
type: "x86_64",
ext: "deb"
)
} } }
}
stage('Build') {
steps { script { dir(env.REPO) {
nix.develop('make build')
} } }
}
stage('Build examples') {
steps { script { dir(env.REPO) {
nix.develop('make build-example')
} } }
}
stage('Package') {
steps { script { dir(env.REPO) {
dir('./scripts/linux') {
nix.develop('./fpm-build.sh', attr: 'fpm')
}
dir('build') {
sh "cp gowaku*.deb ${env.DEB_ARTIFACT}"
}
} } }
}
stage('Parallel Upload') {
parallel {
stage('Archive') {
steps { script {
archiveArtifacts(env.DEB_ARTIFACT.minus("${env.WORKSPACE}/"))
} }
}
stage('Upload') {
steps { script {
env.PKG_URL = s3.uploadArtifact(env.DEB_ARTIFACT)
jenkins.setBuildDesc(x86_64_deb: env.PKG_URL)
} }
}
}
}
}
post {
success { script { github.notifyPR(true) } }
failure { script { github.notifyPR(false) } }
always { cleanWs() }
}
}

View File

@ -1,126 +0,0 @@
library 'status-jenkins-lib@v1.7.0'
pipeline {
agent {
label 'linux && nix-2.11 && x86_64'
}
options {
timestamps()
disableConcurrentBuilds()
/* Prevent Jenkins jobs from running forever */
timeout(time: 30, unit: 'MINUTES')
/* Go requires a certain directory structure */
checkoutToSubdirectory('src/github.com/waku-org/go-waku')
buildDiscarder(logRotator(
numToKeepStr: '10',
daysToKeepStr: '30',
))
}
/* WARNING: Defining parameters here with the ?: trick causes them to remember last value. */
parameters {
booleanParam(
name: 'RACE',
description: 'Run tests with check for race condition.',
defaultValue: getRaceDefault()
)
}
environment {
TARGET = 'tests'
DB_CONT = "go-waku-test-db-${env.EXECUTOR_NUMBER.toInteger() + 1}"
DB_PORT = "${5432 + env.EXECUTOR_NUMBER.toInteger()}"
REPO = "${env.WORKSPACE}/src/github.com/waku-org/go-waku"
GOCACHE = "${env.WORKSPACE_TMP}/go-build"
GOPATH = "${env.WORKSPACE}/go"
PATH = "${env.PATH}:${env.GOPATH}/bin"
/* Necesary to avoid cache poisoning by other builds. */
GOLANGCI_LINT_CACHE = "${env.WORKSPACE_TMP}/golangci-lint"
/* Ganache config */
GANACHE_RPC_PORT = "${8989 + env.EXECUTOR_NUMBER.toInteger()}"
GANACHE_MNEMONIC = 'swim relax risk shy chimney please usual search industry board music segment'
}
stages {
stage('Lint') {
steps { script { dir(env.REPO) {
nix.develop('make lint', pure: false)
} } }
}
stage('Test') {
steps { script { dir(env.REPO) {
if (params.RACE) {
nix.develop('make test-with-race', pure: false)
}else {
nix.develop('make test-ci', pure: false)
}
} } }
}
stage('PostgresSQL') {
environment {
TEST_DB_PORT = "${env.DB_PORT}"
}
steps { script { dir(env.REPO) {
db = docker.image('postgres:9.6-alpine').withRun([
"--name=${DB_CONT}",
"--env=POSTGRES_HOST_AUTH_METHOD=trust",
"--publish=${DB_PORT}:${DB_PORT}",
].join(' '), "-p ${DB_PORT}") { c ->
if (params.RACE) {
nix.develop('make test-postgres-with-race', pure: false)
}else {
nix.develop('make test-postgres', pure: false)
}
}
} } }
post { cleanup { /* Leftover DB containers. */
sh "docker rm ${DB_CONT} || true"
} }
}
stage('Ganache') {
steps { script {
ganache = docker.image(
'trufflesuite/ganache:v7.4.1'
).run(
"-p 127.0.0.1:${env.GANACHE_RPC_PORT}:8545",
"-m='${GANACHE_MNEMONIC}'"
)
} }
}
stage('On-chain tests') {
environment {
GANACHE_NETWORK_RPC_URL = "ws://localhost:${env.GANACHE_RPC_PORT}"
}
steps { script { dir(env.REPO) {
if (params.RACE) {
nix.develop('make test-onchain-with-race', pure: false)
}else {
nix.develop('make test-onchain', pure: false)
}
} } }
}
}
post {
always { script { /* No artifact but a PKG_URL is necessary. */
env.PKG_URL = "${currentBuild.absoluteUrl}consoleText"
} }
success { script { github.notifyPR(true) } }
failure { script { github.notifyPR(false) } }
cleanup { script {
cleanWs()
catchError {
ganache.stop()
}
} }
}
}
def Boolean getRaceDefault() {
return env.JOB_NAME.split('/').contains('race')
}