ci: use Nix shell to provide Android SDK for builds

Installing the SDK via Ansible is prone to error and not exactly
reproduceable. This way we can also track the exact tooling version as
used in Status Mobile app:
https://github.com/status-im/status-mobile/blob/develop/nix/pkgs.nix

Signed-off-by: Jakub Sokołowski <jakub@status.im>

Makefile

@@ -141,9 +141,9 @@ statusgo-ios: ##@cross-compile Build status-go for iOS
 		-target=ios -ldflags="-s -w" \
 		-tags 'nowatchdog $(BUILD_TAGS)' \
 		$(BUILD_FLAGS_MOBILE) \
-		-o build/bin/Statusgo.framework \
+		-o build/bin/Statusgo.xcframework \
 		github.com/status-im/status-go/mobile
-	@echo "iOS framework cross compilation done in build/bin/Statusgo.framework"
+	@echo "iOS framework cross compilation done in build/bin/Statusgo.xcframework"
 
 statusgo-library: ##@cross-compile Build status-go as static library for current platform
 	## cmd/library/README.md explains the magic incantation behind this
@@ -272,8 +272,8 @@ generate: ##@other Regenerate assets and other auto-generated stuff
 prepare-release: clean-release
 	mkdir -p $(RELEASE_DIR)
 	mv build/bin/statusgo.aar $(RELEASE_DIR)/status-go-android.aar
-	zip -r build/bin/Statusgo.framework.zip build/bin/Statusgo.framework
+	zip -r build/bin/Statusgo.xcframework.zip build/bin/Statusgo.xcframework
-	mv build/bin/Statusgo.framework.zip $(RELEASE_DIR)/status-go-ios.zip
+	mv build/bin/Statusgo.xcframework.zip $(RELEASE_DIR)/status-go-ios.zip
 	zip -r $(RELEASE_DIR)/status-go-desktop.zip . -x *.git*
 	${MAKE} clean
 

_assets/ci/Jenkinsfile

@@ -1,4 +1,4 @@
-library 'status-jenkins-lib@v1.3.3'
+library 'status-jenkins-lib@v1.6.4'
 
 pipeline {
   agent { label 'linux' }

_assets/ci/Jenkinsfile.android

@@ -1,7 +1,7 @@
-library 'status-jenkins-lib@v1.3.3'
+library 'status-jenkins-lib@v1.6.4'
 
 pipeline {
-  agent { label 'linux && x86_64 && go-1.18' }
+  agent { label 'linux && x86_64' }
 
   parameters {
     string(
@@ -19,8 +19,6 @@ pipeline {
   options {
     timestamps()
     disableConcurrentBuilds()
-    /* Go requires a certain directory structure */
-    checkoutToSubdirectory('src/github.com/status-im/status-go')
     /* manage how many builds we keep */
     buildDiscarder(logRotator(
       numToKeepStr: '5',
@@ -29,57 +27,43 @@ pipeline {
   }
 
   environment {
-    /* fix for gomobile complaining about missing packages */
-    CGO_ENABLED = "1"
-    /* Other stuff */
     TARGET   = 'android'
-    REPO   = "${env.WORKSPACE}/src/github.com/status-im/status-go"
+    GOPATH   = "${WORKSPACE_TMP}/go"
-    GOPATH = "${env.WORKSPACE}"
+    PATH     = "${PATH}:${GOPATH}/bin"
-    PATH   = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin"
+    REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go"
-    /* Android SDK */
+    ARTIFACT = utils.pkgFilename(name: "status-go", type: "android", ext: "aar", version: null)
-    ANDROID_HOME     = '/usr/lib/android-sdk'
-    ANDROID_SDK_ROOT = '/usr/lib/android-sdk'
-    /* gomobile requires a specific NDK version */
-    ANDROID_NDK      = "${env.NDK_GOMOBILE}"
-    ANDROID_NDK_HOME = "${env.NDK_GOMOBILE}"
   }
 
   stages {
-    stage('Prep') { steps { dir(env.REPO) { script {
+    stage('Setup') {
-      env.ARTIFACT = "${env.WORKSPACE}/" + utils.pkgFilename(
+      steps { /* Go needs to find status-go in GOPATH. */
-        name: "status-go",
+        sh "mkdir -p \$(dirname ${REPO_SRC})"
-        type: "android",
+        sh "ln -s ${WORKSPACE} ${REPO_SRC}"
-        ext: "aar"
+      }
-      )
+    }
-      println("Output: ${env.ARTIFACT}")
-    } } } }
 
-    stage('Setup') { steps { dir(env.REPO) {
+    stage('Compile') {
-      sh 'make setup-build install-modvendor'
+      steps { script {
-    } } }
+        nix.shell('make statusgo-android', pure: false)
-
+        sh "mv build/bin/statusgo.aar ${ARTIFACT}"
-    stage('Vendoring check') { steps { dir(env.REPO) {
-      /* fail build if vendoring hasn't been done */
-      sh 'make vendor'
-      sh 'git diff --exit-code --no-color --stat vendor/'
-    } } }
-
-    stage('Compile') { steps { dir(env.REPO) {
-      sh 'make statusgo-android'
-      sh "cp build/bin/statusgo.aar ${env.ARTIFACT}"
-    } } }
-
-    stage('Archive') { steps {
-      archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/"))
       } }
+    }
 
-    stage('Upload') { steps { script {
+    stage('Archive') {
-      env.PKG_URL = s3.uploadArtifact(env.ARTIFACT)
+      steps { script {
-    } } }
+        archiveArtifacts(ARTIFACT)
+      } }
+    }
+
+    stage('Upload') {
+      steps { script {
+        env.PKG_URL = s3.uploadArtifact(ARTIFACT)
+      } }
+    }
   } // stages
   post {
     success { script { github.notifyPR(true) } }
     failure { script { github.notifyPR(false) } }
-    always { dir(env.REPO) { sh 'make clean' } }
+    cleanup { sh 'make clean' }
   } // post
 } // pipeline

_assets/ci/Jenkinsfile.docker

@@ -1,4 +1,4 @@
-library 'status-jenkins-lib@v1.3.3'
+library 'status-jenkins-lib@v1.6.4'
 
 pipeline {
   agent { label 'linux' }

_assets/ci/Jenkinsfile.ios

@@ -1,4 +1,4 @@
-library 'status-jenkins-lib@v1.3.3'
+library 'status-jenkins-lib@v1.6.4'
 
 pipeline {
   agent { label 'macos && x86_64 && go-1.18' }
@@ -19,8 +19,6 @@ pipeline {
   options {
     timestamps()
     disableConcurrentBuilds()
-    /* Go requires a certain directory structure */
-    checkoutToSubdirectory('src/github.com/status-im/status-go')
     /* manage how many builds we keep */
     buildDiscarder(logRotator(
       numToKeepStr: '5',
@@ -29,48 +27,48 @@ pipeline {
   }
 
   environment {
+    TARGET   = 'ios'
+    GOPATH   = "${WORKSPACE_TMP}/go"
+    PATH     = "${PATH}:${GOPATH}/bin"
+    REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go"
+    ARTIFACT = utils.pkgFilename(name: "status-go", type: "ios", ext: "zip", version: null)
     /* fix for gomobile complaining about missing packages */
     CGO_ENABLED = "1"
-    /* Other stuff */
-    TARGET = 'ios'
-    GOPATH = "${env.WORKSPACE}"
-    REPO   = "${env.WORKSPACE}/src/github.com/status-im/status-go"
-    PATH   = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin"
   }
 
   stages {
-    stage('Prep') { steps { dir(env.REPO) { script {
+    stage('Prep') {
-      env.ARTIFACT = "${env.WORKSPACE}/" + utils.pkgFilename(
+      steps { /* Go needs to find status-go in GOPATH. */
-        name: "status-go",
+        sh "mkdir -p \$(dirname ${REPO_SRC})"
-        type: "ios",
+        sh "ln -s ${WORKSPACE} ${REPO_SRC}"
-        ext: "zip"
+      }
-      )
-      println("Output: ${env.ARTIFACT}")
-    } } } }
-
-    stage('Setup') { steps { dir(env.REPO) {
-      sh 'unset TMPDIR && make setup-build'
-    } } }
-
-    stage('Compile') { steps { dir(env.REPO) {
-      sh 'make statusgo-ios'
-      dir('build/bin') {
-        sh 'zip -r status-go-ios.zip Statusgo.framework'
-        sh "cp status-go-ios.zip ${env.ARTIFACT}"
     }
-    } } }
 
-    stage('Archive') { steps {
+    stage('Compile') {
-      archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/"))
+      steps { script {
+        nix.shell('make statusgo-ios', pure: false)
       } }
+    }
 
-    stage('Upload') { steps { script {
+    stage('Archive') {
-      env.PKG_URL = s3.uploadArtifact(env.ARTIFACT)
+      steps {
-    } } }
+        dir('build/bin') {
+          sh 'zip -r status-go-ios.zip Statusgo.xcframework'
+          sh "mv status-go-ios.zip ${WORKSPACE}/${ARTIFACT}"
+        }
+        archiveArtifacts(ARTIFACT)
+      }
+    }
+
+    stage('Upload') {
+      steps { script {
+        env.PKG_URL = s3.uploadArtifact(ARTIFACT)
+      } }
+    }
   } // stages
   post {
     success { script { github.notifyPR(true) } }
     failure { script { github.notifyPR(false) } }
-    always { dir(env.REPO) { sh 'make clean' } }
+    cleanup { sh 'make clean' }
   } // post
 } // pipeline

_assets/ci/Jenkinsfile.linux

@@ -1,4 +1,4 @@
-library 'status-jenkins-lib@v1.3.3'
+library 'status-jenkins-lib@v1.6.4'
 
 pipeline {
   agent { label 'linux && x86_64 && go-1.18' }
@@ -19,8 +19,6 @@ pipeline {
   options {
     timestamps()
     disableConcurrentBuilds()
-    /* Go requires a certain directory structure */
-    checkoutToSubdirectory('src/github.com/status-im/status-go')
     /* manage how many builds we keep */
     buildDiscarder(logRotator(
       numToKeepStr: '5',
@@ -30,45 +28,43 @@ pipeline {
 
   environment {
     TARGET   = 'linux'
-    REPO   = "${env.WORKSPACE}/src/github.com/status-im/status-go"
+    GOPATH   = "${WORKSPACE_TMP}/go"
-    GOPATH = "${env.WORKSPACE}"
+    PATH     = "${PATH}:${GOPATH}/bin"
-    PATH   = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin"
+    REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go"
+    ARTIFACT = utils.pkgFilename(name: "status-go", type: "desktop", ext: "zip", version: null)
   }
 
   stages {
-    stage('Prep') { steps { dir(env.REPO) { script {
+    stage('Setup') {
-      env.ARTIFACT = "${env.WORKSPACE}/" + utils.pkgFilename(
+      steps { /* Go needs to find status-go in GOPATH. */
-        name: "status-go",
+        sh "mkdir -p \$(dirname ${REPO_SRC})"
-        type: "desktop",
+        sh "ln -s ${WORKSPACE} ${REPO_SRC}"
-        ext: "zip"
+      }
-      )
+    }
-      println("Output: ${env.ARTIFACT}")
-    } } } }
-
-    stage('Setup') { steps { dir(env.REPO) {
-      sh 'make setup-build'
-    } } }
 
     /* Sanity-check C bindings */
-    stage('Sanity check bindings') { steps { dir(env.REPO) {
+    stage('Bindings Check') {
+      steps {
         sh 'make statusgo-library'
-    } } }
+      }
+    }
 
-    stage('Compress') { steps { dir(env.REPO) {
+    stage('Archive') {
-      sh "zip -q -r ${env.ARTIFACT} . -x *.git"
+      steps {
-    } } }
+        sh "zip -q -r ${ARTIFACT} build/bin"
+        archiveArtifacts(ARTIFACT)
+      }
+    }
 
-    stage('Archive') { steps {
+    stage('Upload') {
-      archiveArtifacts(env.ARTIFACT.minus("${env.WORKSPACE}/"))
+      steps { script {
+        env.PKG_URL = s3.uploadArtifact(ARTIFACT)
       } }
-
+    }
-    stage('Upload') { steps { script {
-      env.PKG_URL = s3.uploadArtifact(env.ARTIFACT)
-    } } }
   } // stages
   post {
     success { script { github.notifyPR(true) } }
     failure { script { github.notifyPR(false) } }
-    always { dir(env.REPO) { sh 'make clean' } }
+    cleanup { sh 'make clean' }
   } // post
 } // pipeline

_assets/ci/Jenkinsfile.tests

@@ -1,4 +1,4 @@
-library 'status-jenkins-lib@v1.3.3'
+library 'status-jenkins-lib@v1.6.4'
 
 pipeline {
   agent { label 'linux && x86_64 && go-1.18' }
@@ -14,8 +14,6 @@ pipeline {
   options {
     timestamps()
     disableConcurrentBuilds()
-    /* Go requires a certain directory structure */
-    checkoutToSubdirectory('src/github.com/status-im/status-go')
     /* manage how many builds we keep */
     buildDiscarder(logRotator(
       numToKeepStr: '5',
@@ -25,42 +23,50 @@ pipeline {
 
   environment {
     TARGET   = 'linux'
-    GOPATH = "${env.WORKSPACE}"
+    GOPATH   = "${WORKSPACE_TMP}/go"
-    PATH   = "/usr/local/go/bin:${env.PATH}:${env.GOPATH}/bin"
+    PATH     = "${PATH}:${GOPATH}/bin"
-    REPO   = 'src/github.com/status-im/status-go'
+    REPO_SRC = "${GOPATH}/src/github.com/status-im/status-go"
   }
 
   stages {
-    stage('Prep') { steps { dir(env.REPO) { script {
+    stage('Prep') {
-      println("Version:    ${utils.getVersion()}")
+      steps { /* Go needs to find status-go in GOPATH. */
-      println("Git Branch: ${utils.branchName()}")
+        sh "mkdir -p \$(dirname ${REPO_SRC})"
-      println("Git Commit: ${utils.gitCommit()}")
+        sh "ln -s ${WORKSPACE} ${REPO_SRC}"
-    } } } }
+      }
+    }
 
-    stage('Setup') { steps { dir(env.REPO) {
+    stage('Vendor Check') {
-      sh 'make setup-build install-modvendor'
+      steps { script {
-    } } }
+        nix.shell('make install-modvendor', pure: false)
-
+        nix.shell('make vendor', pure: false)
-    stage('Vendoring check') { steps { dir(env.REPO) {
         /* fail build if vendoring hasn't been done */
-      sh 'make vendor'
+        nix.shell('git diff --exit-code --no-color --stat vendor/')
-      sh 'git diff --exit-code --no-color --stat vendor/'
+      } }
-    } } }
+    }
 
-    stage('Lint') { steps { dir(env.REPO) {
-      sh 'make lint'
-    } } }
 
-    stage('Canary') { steps { dir(env.REPO) {
+    stage('Lint') {
-      sh 'make canary-test'
+      steps { script {
-    } } }
+        nix.shell('make install-lint', pure: false)
+        nix.shell('make lint', pure: false)
+      } }
+    }
 
-    stage('Unit Tests') { steps { script { dir(env.REPO) {
+    stage('Canary') {
+      steps { script {
+        nix.shell('make canary-test', pure: false)
+      } }
+    }
+
+    stage('Unit Tests') {
+      steps { script {
         docker.image('postgres:9.6-alpine').withRun(
           '-e POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432'
         ) { c ->
-        sh 'make test-unit V=1'
+          nix.shell('make test-unit V=1', pure: false)
+        }
+      } }
     }
-    } } } }
   } // stages
 } // pipeline

nix.conf

@@ -0,0 +1,10 @@
+# NOTE: If you are in Asia you might want to add https://nix-cache-cn.status.im/ to extra-substituters
+extra-substituters = https://nix-cache.status.im/
+substituters = https://cache.nixos.org/
+trusted-public-keys = nix-cache.status.im-1:x/93lOfLU+duPplwMSBR+OlY4+mo+dCN7n0mr4oPwgY= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+# Some downloads are multiple GB, default is 5 minutes
+stalled-download-timeout = 3600
+connect-timeout = 10
+max-jobs = auto
+# Some builds on MacOS have issue with sandbox so they are disabled with __noChroot.
+sandbox = relaxed

shell.nix

@@ -0,0 +1,52 @@
+{
+  /* This should match Nixpkgs commit in status-mobile. */
+  source ? builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/579238da5f431b7833a9f0681663900aaf0dd1e8.zip";
+    sha256 = "sha256:0a77c8fq4145k0zdmsda9cmhfw84ipf9nhvvn0givzhza1500g3h";
+  },
+  pkgs ? import (source){
+    config = {
+      allowUnfree = true;
+      android_sdk.accept_license = true;
+    };
+    overlays = [
+      (self: super: {
+        androidPkgs = pkgs.androidenv.composeAndroidPackages {
+          toolsVersion = "26.1.1";
+          platformToolsVersion = "33.0.2";
+          buildToolsVersions = [ "31.0.0" ];
+          platformVersions = [ "31" ];
+          cmakeVersions = [ "3.18.1" ];
+          ndkVersion = "22.1.7171670";
+          includeNDK = true;
+          includeExtras = [
+            "extras;android;m2repository"
+            "extras;google;m2repository"
+          ];
+        };
+      })
+    ];
+  }
+}:
+
+pkgs.mkShell {
+  name = "status-go-shell";
+
+  buildInputs = with pkgs; [
+    git jq which
+    go_1_18 gopls go-bindata
+    mockgen protobuf3_17 protoc-gen-go
+    (gomobile.override { xcodeWrapperArgs = { version = "13.4.1"; }; })
+  ] ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [
+    (xcodeenv.composeXcodeWrapper { version = "13.4.1"; })
+  ];
+
+  shellHook = let
+    androidSdk = pkgs.androidPkgs.androidsdk;
+  in ''
+    ANDROID_HOME=${androidSdk}/libexec/android-sdk
+    ANDROID_SDK_ROOT=$ANDROID_HOME
+    ANDROID_NDK=${androidSdk}/libexec/android-sdk/ndk-bundle
+    ANDROID_NDK_HOME=$ANDROID_NDK
+  '';
+}