From 9dd80e7f1e7e5c5a9aa2ee2b35e67445ca7d4be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Soko=C5=82owski?= Date: Fri, 2 Dec 2022 19:03:35 +0100 Subject: [PATCH] ci: use Nix shell to provide Android SDK for builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installing the SDK via Ansible is prone to error and not exactly reproduceable. This way we can also track the exact tooling version as used in Status Mobile app: https://github.com/status-im/status-mobile/blob/develop/nix/pkgs.nix Signed-off-by: Jakub SokoĊ‚owski --- Makefile | 8 ++-- _assets/ci/Jenkinsfile | 2 +- _assets/ci/Jenkinsfile.android | 76 +++++++++++++------------------- _assets/ci/Jenkinsfile.docker | 2 +- _assets/ci/Jenkinsfile.ios | 66 ++++++++++++++-------------- _assets/ci/Jenkinsfile.linux | 62 ++++++++++++-------------- _assets/ci/Jenkinsfile.tests | 80 ++++++++++++++++++---------------- nix.conf | 10 +++++ shell.nix | 52 ++++++++++++++++++++++ 9 files changed, 202 insertions(+), 156 deletions(-) create mode 100644 nix.conf create mode 100644 shell.nix diff --git a/Makefile b/Makefile index 9772e3887..d7c11d122 100644 --- a/Makefile +++ b/Makefile @@ -141,9 +141,9 @@ statusgo-ios: ##@cross-compile Build status-go for iOS -target=ios -ldflags="-s -w" \ -tags 'nowatchdog $(BUILD_TAGS)' \ $(BUILD_FLAGS_MOBILE) \ - -o build/bin/Statusgo.framework \ + -o build/bin/Statusgo.xcframework \ github.com/status-im/status-go/mobile - @echo "iOS framework cross compilation done in build/bin/Statusgo.framework" + @echo "iOS framework cross compilation done in build/bin/Statusgo.xcframework" statusgo-library: ##@cross-compile Build status-go as static library for current platform ## cmd/library/README.md explains the magic incantation behind this @@ -272,8 +272,8 @@ generate: ##@other Regenerate assets and other auto-generated stuff prepare-release: clean-release mkdir -p $(RELEASE_DIR) mv build/bin/statusgo.aar $(RELEASE_DIR)/status-go-android.aar - zip -r build/bin/Statusgo.framework.zip build/bin/Statusgo.framework - mv build/bin/Statusgo.framework.zip $(RELEASE_DIR)/status-go-ios.zip + zip -r build/bin/Statusgo.xcframework.zip build/bin/Statusgo.xcframework + mv build/bin/Statusgo.xcframework.zip $(RELEASE_DIR)/status-go-ios.zip zip -r $(RELEASE_DIR)/status-go-desktop.zip . -x *.git* ${MAKE} clean diff --git a/_assets/ci/Jenkinsfile b/_assets/ci/Jenkinsfile index f19b0ad5d..e562816af 100644 --- a/_assets/ci/Jenkinsfile +++ b/_assets/ci/Jenkinsfile @@ -1,4 +1,4 @@ -library 'status-jenkins-lib@v1.3.3' +library 'status-jenkins-lib@v1.6.4' pipeline { agent { label 'linux' } diff --git a/_assets/ci/Jenkinsfile.android b/_assets/ci/Jenkinsfile.android index 1ba25af56..6631154c0 100644 --- a/_assets/ci/Jenkinsfile.android +++ b/_assets/ci/Jenkinsfile.android @@ -1,7 +1,7 @@ -library 'status-jenkins-lib@v1.3.3' +library 'status-jenkins-lib@v1.6.4' pipeline { - agent { label 'linux && x86_64 && go-1.18' } + agent { label 'linux && x86_64' } parameters { string( @@ -19,8 +19,6 @@ pipeline { options { timestamps() disableConcurrentBuilds() - /* Go requires a certain directory structure */ - checkoutToSubdirectory('src/github.com/status-im/status-go') /* manage how many builds we keep */ buildDiscarder(logRotator( numToKeepStr: '5', @@ -29,57 +27,43 @@ pipeline { } environment { - /* fix for gomobile complaining about missing packages */ - CGO_ENABLED = "1" - /* Other stuff */ - TARGET = 'android' - REPO = "${env.WORKSPACE}/src/github.com/status-im/status-go" - GOPATH = "${env.WORKSPACE}" - PATH = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin" - /* Android SDK */ - ANDROID_HOME = '/usr/lib/android-sdk' - ANDROID_SDK_ROOT = '/usr/lib/android-sdk' - /* gomobile requires a specific NDK version */ - ANDROID_NDK = "${env.NDK_GOMOBILE}" - ANDROID_NDK_HOME = "${env.NDK_GOMOBILE}" + TARGET = 'android' + GOPATH = "${WORKSPACE_TMP}/go" + PATH = "${PATH}:${GOPATH}/bin" + REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go" + ARTIFACT = utils.pkgFilename(name: "status-go", type: "android", ext: "aar", version: null) } stages { - stage('Prep') { steps { dir(env.REPO) { script { - env.ARTIFACT = "${env.WORKSPACE}/" + utils.pkgFilename( - name: "status-go", - type: "android", - ext: "aar" - ) - println("Output: ${env.ARTIFACT}") - } } } } + stage('Setup') { + steps { /* Go needs to find status-go in GOPATH. */ + sh "mkdir -p \$(dirname ${REPO_SRC})" + sh "ln -s ${WORKSPACE} ${REPO_SRC}" + } + } - stage('Setup') { steps { dir(env.REPO) { - sh 'make setup-build install-modvendor' - } } } + stage('Compile') { + steps { script { + nix.shell('make statusgo-android', pure: false) + sh "mv build/bin/statusgo.aar ${ARTIFACT}" + } } + } - stage('Vendoring check') { steps { dir(env.REPO) { - /* fail build if vendoring hasn't been done */ - sh 'make vendor' - sh 'git diff --exit-code --no-color --stat vendor/' - } } } + stage('Archive') { + steps { script { + archiveArtifacts(ARTIFACT) + } } + } - stage('Compile') { steps { dir(env.REPO) { - sh 'make statusgo-android' - sh "cp build/bin/statusgo.aar ${env.ARTIFACT}" - } } } - - stage('Archive') { steps { - archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/")) - } } - - stage('Upload') { steps { script { - env.PKG_URL = s3.uploadArtifact(env.ARTIFACT) - } } } + stage('Upload') { + steps { script { + env.PKG_URL = s3.uploadArtifact(ARTIFACT) + } } + } } // stages post { success { script { github.notifyPR(true) } } failure { script { github.notifyPR(false) } } - always { dir(env.REPO) { sh 'make clean' } } + cleanup { sh 'make clean' } } // post } // pipeline diff --git a/_assets/ci/Jenkinsfile.docker b/_assets/ci/Jenkinsfile.docker index 7cb3884c5..c130045b4 100644 --- a/_assets/ci/Jenkinsfile.docker +++ b/_assets/ci/Jenkinsfile.docker @@ -1,4 +1,4 @@ -library 'status-jenkins-lib@v1.3.3' +library 'status-jenkins-lib@v1.6.4' pipeline { agent { label 'linux' } diff --git a/_assets/ci/Jenkinsfile.ios b/_assets/ci/Jenkinsfile.ios index 083b1998f..12e5b0ce9 100644 --- a/_assets/ci/Jenkinsfile.ios +++ b/_assets/ci/Jenkinsfile.ios @@ -1,4 +1,4 @@ -library 'status-jenkins-lib@v1.3.3' +library 'status-jenkins-lib@v1.6.4' pipeline { agent { label 'macos && x86_64 && go-1.18' } @@ -19,8 +19,6 @@ pipeline { options { timestamps() disableConcurrentBuilds() - /* Go requires a certain directory structure */ - checkoutToSubdirectory('src/github.com/status-im/status-go') /* manage how many builds we keep */ buildDiscarder(logRotator( numToKeepStr: '5', @@ -29,48 +27,48 @@ pipeline { } environment { + TARGET = 'ios' + GOPATH = "${WORKSPACE_TMP}/go" + PATH = "${PATH}:${GOPATH}/bin" + REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go" + ARTIFACT = utils.pkgFilename(name: "status-go", type: "ios", ext: "zip", version: null) /* fix for gomobile complaining about missing packages */ CGO_ENABLED = "1" - /* Other stuff */ - TARGET = 'ios' - GOPATH = "${env.WORKSPACE}" - REPO = "${env.WORKSPACE}/src/github.com/status-im/status-go" - PATH = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin" } stages { - stage('Prep') { steps { dir(env.REPO) { script { - env.ARTIFACT = "${env.WORKSPACE}/" + utils.pkgFilename( - name: "status-go", - type: "ios", - ext: "zip" - ) - println("Output: ${env.ARTIFACT}") - } } } } - - stage('Setup') { steps { dir(env.REPO) { - sh 'unset TMPDIR && make setup-build' - } } } - - stage('Compile') { steps { dir(env.REPO) { - sh 'make statusgo-ios' - dir('build/bin') { - sh 'zip -r status-go-ios.zip Statusgo.framework' - sh "cp status-go-ios.zip ${env.ARTIFACT}" + stage('Prep') { + steps { /* Go needs to find status-go in GOPATH. */ + sh "mkdir -p \$(dirname ${REPO_SRC})" + sh "ln -s ${WORKSPACE} ${REPO_SRC}" } - } } } + } - stage('Archive') { steps { - archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/")) - } } + stage('Compile') { + steps { script { + nix.shell('make statusgo-ios', pure: false) + } } + } - stage('Upload') { steps { script { - env.PKG_URL = s3.uploadArtifact(env.ARTIFACT) - } } } + stage('Archive') { + steps { + dir('build/bin') { + sh 'zip -r status-go-ios.zip Statusgo.xcframework' + sh "mv status-go-ios.zip ${WORKSPACE}/${ARTIFACT}" + } + archiveArtifacts(ARTIFACT) + } + } + + stage('Upload') { + steps { script { + env.PKG_URL = s3.uploadArtifact(ARTIFACT) + } } + } } // stages post { success { script { github.notifyPR(true) } } failure { script { github.notifyPR(false) } } - always { dir(env.REPO) { sh 'make clean' } } + cleanup { sh 'make clean' } } // post } // pipeline diff --git a/_assets/ci/Jenkinsfile.linux b/_assets/ci/Jenkinsfile.linux index e4bccacf3..c6ea91600 100644 --- a/_assets/ci/Jenkinsfile.linux +++ b/_assets/ci/Jenkinsfile.linux @@ -1,4 +1,4 @@ -library 'status-jenkins-lib@v1.3.3' +library 'status-jenkins-lib@v1.6.4' pipeline { agent { label 'linux && x86_64 && go-1.18' } @@ -19,8 +19,6 @@ pipeline { options { timestamps() disableConcurrentBuilds() - /* Go requires a certain directory structure */ - checkoutToSubdirectory('src/github.com/status-im/status-go') /* manage how many builds we keep */ buildDiscarder(logRotator( numToKeepStr: '5', @@ -29,46 +27,44 @@ pipeline { } environment { - TARGET = 'linux' - REPO = "${env.WORKSPACE}/src/github.com/status-im/status-go" - GOPATH = "${env.WORKSPACE}" - PATH = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin" + TARGET = 'linux' + GOPATH = "${WORKSPACE_TMP}/go" + PATH = "${PATH}:${GOPATH}/bin" + REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go" + ARTIFACT = utils.pkgFilename(name: "status-go", type: "desktop", ext: "zip", version: null) } stages { - stage('Prep') { steps { dir(env.REPO) { script { - env.ARTIFACT = "${env.WORKSPACE}/" + utils.pkgFilename( - name: "status-go", - type: "desktop", - ext: "zip" - ) - println("Output: ${env.ARTIFACT}") - } } } } - - stage('Setup') { steps { dir(env.REPO) { - sh 'make setup-build' - } } } + stage('Setup') { + steps { /* Go needs to find status-go in GOPATH. */ + sh "mkdir -p \$(dirname ${REPO_SRC})" + sh "ln -s ${WORKSPACE} ${REPO_SRC}" + } + } /* Sanity-check C bindings */ - stage('Sanity check bindings') { steps { dir(env.REPO) { - sh 'make statusgo-library' - } } } + stage('Bindings Check') { + steps { + sh 'make statusgo-library' + } + } - stage('Compress') { steps { dir(env.REPO) { - sh "zip -q -r ${env.ARTIFACT} . -x *.git" - } } } + stage('Archive') { + steps { + sh "zip -q -r ${ARTIFACT} build/bin" + archiveArtifacts(ARTIFACT) + } + } - stage('Archive') { steps { - archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/")) - } } - - stage('Upload') { steps { script { - env.PKG_URL = s3.uploadArtifact(env.ARTIFACT) - } } } + stage('Upload') { + steps { script { + env.PKG_URL = s3.uploadArtifact(ARTIFACT) + } } + } } // stages post { success { script { github.notifyPR(true) } } failure { script { github.notifyPR(false) } } - always { dir(env.REPO) { sh 'make clean' } } + cleanup { sh 'make clean' } } // post } // pipeline diff --git a/_assets/ci/Jenkinsfile.tests b/_assets/ci/Jenkinsfile.tests index 99cfc05bc..2df6584b2 100644 --- a/_assets/ci/Jenkinsfile.tests +++ b/_assets/ci/Jenkinsfile.tests @@ -1,4 +1,4 @@ -library 'status-jenkins-lib@v1.3.3' +library 'status-jenkins-lib@v1.6.4' pipeline { agent { label 'linux && x86_64 && go-1.18' } @@ -14,8 +14,6 @@ pipeline { options { timestamps() disableConcurrentBuilds() - /* Go requires a certain directory structure */ - checkoutToSubdirectory('src/github.com/status-im/status-go') /* manage how many builds we keep */ buildDiscarder(logRotator( numToKeepStr: '5', @@ -24,43 +22,51 @@ pipeline { } environment { - TARGET = 'linux' - GOPATH = "${env.WORKSPACE}" - PATH = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin" - REPO = 'src/github.com/status-im/status-go' + TARGET = 'linux' + GOPATH = "${WORKSPACE_TMP}/go" + PATH = "${PATH}:${GOPATH}/bin" + REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go" } stages { - stage('Prep') { steps { dir(env.REPO) { script { - println("Version: ${utils.getVersion()}") - println("Git Branch: ${utils.branchName()}") - println("Git Commit: ${utils.gitCommit()}") - } } } } - - stage('Setup') { steps { dir(env.REPO) { - sh 'make setup-build install-modvendor' - } } } - - stage('Vendoring check') { steps { dir(env.REPO) { - /* fail build if vendoring hasn't been done */ - sh 'make vendor' - sh 'git diff --exit-code --no-color --stat vendor/' - } } } - - stage('Lint') { steps { dir(env.REPO) { - sh 'make lint' - } } } - - stage('Canary') { steps { dir(env.REPO) { - sh 'make canary-test' - } } } - - stage('Unit Tests') { steps { script { dir(env.REPO) { - docker.image('postgres:9.6-alpine').withRun( - '-e POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432' - ) { c -> - sh 'make test-unit V=1' + stage('Prep') { + steps { /* Go needs to find status-go in GOPATH. */ + sh "mkdir -p \$(dirname ${REPO_SRC})" + sh "ln -s ${WORKSPACE} ${REPO_SRC}" } - } } } } + } + + stage('Vendor Check') { + steps { script { + nix.shell('make install-modvendor', pure: false) + nix.shell('make vendor', pure: false) + /* fail build if vendoring hasn't been done */ + nix.shell('git diff --exit-code --no-color --stat vendor/') + } } + } + + + stage('Lint') { + steps { script { + nix.shell('make install-lint', pure: false) + nix.shell('make lint', pure: false) + } } + } + + stage('Canary') { + steps { script { + nix.shell('make canary-test', pure: false) + } } + } + + stage('Unit Tests') { + steps { script { + docker.image('postgres:9.6-alpine').withRun( + '-e POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432' + ) { c -> + nix.shell('make test-unit V=1', pure: false) + } + } } + } } // stages } // pipeline diff --git a/nix.conf b/nix.conf new file mode 100644 index 000000000..8d88f27a4 --- /dev/null +++ b/nix.conf @@ -0,0 +1,10 @@ +# NOTE: If you are in Asia you might want to add https://nix-cache-cn.status.im/ to extra-substituters +extra-substituters = https://nix-cache.status.im/ +substituters = https://cache.nixos.org/ +trusted-public-keys = nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +# Some downloads are multiple GB, default is 5 minutes +stalled-download-timeout = 3600 +connect-timeout = 10 +max-jobs = auto +# Some builds on MacOS have issue with sandbox so they are disabled with __noChroot. +sandbox = relaxed diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000..48e409881 --- /dev/null +++ b/shell.nix @@ -0,0 +1,52 @@ +{ + /* This should match Nixpkgs commit in status-mobile. */ + source ? builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/579238da5f431b7833a9f0681663900aaf0dd1e8.zip"; + sha256 = "sha256:0a77c8fq4145k0zdmsda9cmhfw84ipf9nhvvn0givzhza1500g3h"; + }, + pkgs ? import (source){ + config = { + allowUnfree = true; + android_sdk.accept_license = true; + }; + overlays = [ + (self: super: { + androidPkgs = pkgs.androidenv.composeAndroidPackages { + toolsVersion = "26.1.1"; + platformToolsVersion = "33.0.2"; + buildToolsVersions = [ "31.0.0" ]; + platformVersions = [ "31" ]; + cmakeVersions = [ "3.18.1" ]; + ndkVersion = "22.1.7171670"; + includeNDK = true; + includeExtras = [ + "extras;android;m2repository" + "extras;google;m2repository" + ]; + }; + }) + ]; + } +}: + +pkgs.mkShell { + name = "status-go-shell"; + + buildInputs = with pkgs; [ + git jq which + go_1_18 gopls go-bindata + mockgen protobuf3_17 protoc-gen-go + (gomobile.override { xcodeWrapperArgs = { version = "13.4.1"; }; }) + ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ + (xcodeenv.composeXcodeWrapper { version = "13.4.1"; }) + ]; + + shellHook = let + androidSdk = pkgs.androidPkgs.androidsdk; + in '' + ANDROID_HOME=${androidSdk}/libexec/android-sdk + ANDROID_SDK_ROOT=$ANDROID_HOME + ANDROID_NDK=${androidSdk}/libexec/android-sdk/ndk-bundle + ANDROID_NDK_HOME=$ANDROID_NDK + ''; +}