Integrate status-go Nimbus build

Signed-off-by: Pedro Pombeiro <noreply@pedro.pombei.ro>

.env

@@ -21,3 +21,4 @@ PARTITIONED_TOPIC=0
 CONTRACT_NODES=1
 MOBILE_UI_FOR_DESKTOP=1
 STATUS_GO_PROTOCOL=0
+STATUS_GO_ENABLE_NIMBUS=0

.env.jenkins

@@ -20,3 +20,4 @@ RPC_NETWORKS_ONLY=0
 PARTITIONED_TOPIC=0
 CONTRACT_NODES=1
 MOBILE_UI_FOR_DESKTOP=1
+STATUS_GO_ENABLE_NIMBUS=0

Makefile

@@ -40,7 +40,7 @@ export REACT_SERVER_PORT ?= 5001
 # Our custom config is located in nix/nix.conf
 export NIX_CONF_DIR = $(PWD)/nix
 # Defines which variables will be kept for Nix pure shell, use semicolon as divider
-export _NIX_KEEP ?= TMPDIR,BUILD_ENV,STATUS_GO_SRC_OVERRIDE
+export _NIX_KEEP ?= TMPDIR,BUILD_ENV,STATUS_GO_SRC_OVERRIDE,NIMBUS_SRC_OVERRIDE
 export _NIX_ROOT = /nix
 # legacy TARGET_OS variable support
 ifdef TARGET_OS

nix/README.md

@@ -62,6 +62,11 @@ or for a one-off build:
 make release-android STATUS_GO_SRC_OVERRIDE=$GOPATH/src/github.com/status-im/status-go
 ```
 
+## Using a local Nimbus repository
+
+If you need to use a locally checked-out Nimbus repository as a dependency of status-go, you can achieve that by defining the `NIMBUS_SRC_OVERRIDE`
+environment variable, in the same way as the previous point for local status-go repositories.
+
 ## Known Issues
 
 ### MacOS 10.15 "Catalina"

nix/mobile/android/targets/release-android.nix

@@ -14,6 +14,8 @@ let
   inherit (lib) attrByPath hasAttrByPath optionalAttrs;
   env' = env // optionalAttrs (hasAttrByPath ["status-im" "status-go" "src-override"] config) {
     STATUS_GO_SRC_OVERRIDE = config.status-im.status-go.src-override;
+  } // optionalAttrs (hasAttrByPath ["status-im" "nimbus" "src-override"] config) {
+    NIMBUS_SRC_OVERRIDE = config.status-im.nimbus.src-override;
   };
   inherit (config.status-im) build-type;
   inherit (config.status-im.status-react) build-number;

nix/scripts/shell.sh

@@ -55,6 +55,9 @@ config=''
 if [ -n "${STATUS_GO_SRC_OVERRIDE}" ]; then
   config+="status-im.status-go.src-override=\"${STATUS_GO_SRC_OVERRIDE}\";"
 fi
+if [ -n "${NIMBUS_SRC_OVERRIDE}" ]; then
+  config+="status-im.nimbus.src-override=\"${NIMBUS_SRC_OVERRIDE}\";"
+fi
 config+="status-im.build-type=\"${BUILD_TYPE}\";"
 
 if [ -n "$config" ]; then

nix/status-go/build-mobile-status-go.nix

@@ -1,5 +1,5 @@
 { stdenv, utils, callPackage,
-  buildGoPackage, go, gomobile, openjdk, xcodeWrapper }:
+  buildGoPackage, go, gomobile, androidPkgs, openjdk, unzip, zip, xcodeWrapper }:
 
 { owner, repo, rev, cleanVersion, goPackagePath, src, host,
 
@@ -8,7 +8,7 @@
   targetConfig } @ args':
 
 let
-  inherit (stdenv.lib) concatStringsSep makeBinPath optional;
+  inherit (stdenv.lib) concatStringsSep makeBinPath optional optionals;
 
   buildStatusGo = callPackage ./build-status-go.nix {
     inherit buildGoPackage go xcodeWrapper utils;
@@ -19,44 +19,83 @@ let
     "targetConfig" "goBuildFlags" "goBuildLdFlags"
   ];
 
-  buildStatusGoMobileLib = buildStatusGo (args // {
+  buildStatusGoMobileLib =
-    nativeBuildInputs = [ gomobile ] ++ optional (targetConfig.name == "android") openjdk;
+    let
+      inherit (stdenv.lib) concatStrings mapAttrsToList optionalString;
+    in buildStatusGo (args // {
+      nativeBuildInputs = [ gomobile unzip zip ] ++ optional (targetConfig.name == "android") openjdk;
 
-    buildMessage = "Building mobile library for ${targetConfig.name}";
+      buildMessage = "Building mobile library for ${targetConfig.name}";
-    # Build mobile libraries
+      # Build mobile libraries
-    buildPhase =
+      buildPhase =
-      let
+        let
-        NIX_GOWORKDIR = "$NIX_BUILD_TOP/go-build";
+          NIX_GOWORKDIR = "$NIX_BUILD_TOP/go-build";
-        CGO_LDFLAGS = concatStringsSep " " goBuildLdFlags;
+          CGO_LDFLAGS = concatStringsSep " " (goBuildLdFlags ++ [ "-extldflags=-Wl,--allow-multiple-definition" ]);
-      in with targetConfig; ''
+          nimbusBridgeVendorDir = "$NIX_BUILD_TOP/go/src/${goPackagePath}/vendor/${goPackagePath}/eth-node/bridge/nimbus";
-      mkdir ${NIX_GOWORKDIR}
+        in ''
+        mkdir ${NIX_GOWORKDIR}
 
-      export GO111MODULE=off
+        export GO111MODULE=off
-      export GOPATH=${gomobile.dev}:$GOPATH
+        export GOPATH=${gomobile.dev}:$GOPATH
-      export PATH=${makeBinPath [ gomobile.bin ]}:$PATH
+        export PATH=${makeBinPath [ gomobile.bin ]}:$PATH
-      export NIX_GOWORKDIR=${NIX_GOWORKDIR}
+        export NIX_GOWORKDIR=${NIX_GOWORKDIR}
-      export ${concatStringsSep " " envVars}
+        export ${concatStringsSep " " targetConfig.envVars}
-      gomobile bind \
-        -target=${name} \
-        -ldflags='${CGO_LDFLAGS}' \
-        ${concatStringsSep " " gomobileExtraFlags} \
-        ${goBuildFlags} \
-        -o ${outputFileName} \
-        ${goPackagePath}/mobile
 
-      rm -rf ${NIX_GOWORKDIR}
+        # Build the Go library using gomobile for each of the configured platforms
-    '';
+        ${concatStrings (mapAttrsToList (_: platformConfig: ''
 
-    installPhase = ''
+          ${optionalString platformConfig.linkNimbus ''
-      mkdir -p $out/lib
+          # Copy the Nimbus API artifacts to the expected vendor location
-      mv ${targetConfig.outputFileName} $out/lib/
+          cp ${platformConfig.nimbus}/{include/*,lib/libnimbus.a} ${nimbusBridgeVendorDir}
-    '';
+          chmod +w ${nimbusBridgeVendorDir}/libnimbus.{a,h}
+          ''}
 
-    outputs = [ "out" ];
+          echo
+          echo "Building for target ${platformConfig.gomobileTarget}"
+          gomobile bind \
+            -target=${platformConfig.gomobileTarget} \
+            -ldflags="${CGO_LDFLAGS}" \
+            ${concatStringsSep " " targetConfig.gomobileExtraFlags} \
+            ${goBuildFlags} \
+            -o ${platformConfig.outputFileName} \
+            ${goPackagePath}/mobile
 
-    meta = {
+          ${optionalString platformConfig.linkNimbus ''
-      platforms = with stdenv.lib.platforms; linux ++ darwin;
+          rm ${nimbusBridgeVendorDir}/libnimbus.{a,h}
-    };
+          ''}
-  });
+          '') targetConfig.platforms)
+        }
+
+        if [ "${targetConfig.name}" = 'android' ]; then
+          # Merge the platform-specific .aar files into a single one
+          local mergeDir='.aar'
+          mkdir $mergeDir
+          ${concatStrings (mapAttrsToList (_: platformConfig: ''
+            unzip -d $mergeDir -q -n -u ${platformConfig.outputFileName}
+            rm ${platformConfig.outputFileName}
+          '') targetConfig.platforms)}
+          pushd $mergeDir > /dev/null
+            zip -r -o ../${targetConfig.outputFileName} *
+          popd > /dev/null
+          rm -rf $mergeDir
+          unzip -l ${targetConfig.outputFileName}
+        fi
+
+        # TODO: Merge iOS packages when linking with libnimbus.a
+
+        rm -rf ${NIX_GOWORKDIR}
+      '';
+
+      installPhase = ''
+        mkdir -p $out/lib
+        mv ${targetConfig.outputFileName} $out/lib/
+      '';
+
+      outputs = [ "out" ];
+
+      meta = {
+        platforms = with stdenv.lib.platforms; linux ++ darwin;
+      };
+    });
 
 in buildStatusGoMobileLib

nix/status-go/build-status-go.nix

@@ -2,6 +2,7 @@
 
 { owner, repo, rev, cleanVersion, goPackagePath, src, host,
   nativeBuildInputs ? [],
+  buildInputs ? [],
   buildPhase, buildMessage,
   installPhase ? "",
   postInstall ? "",
@@ -24,6 +25,7 @@ let
     nativeBuildInputs =
       nativeBuildInputs ++
       lib.optional isDarwin xcodeWrapper;
+    inherit buildInputs;
 
     # Fixes Cgo related build failures (see https://github.com/NixOS/nixpkgs/issues/25959 )
     hardeningDisable = [ "fortify" ];

nix/status-go/default.nix

@@ -3,14 +3,17 @@
 
 let
   inherit (stdenv.lib)
-    catAttrs concatStrings fileContents importJSON makeBinPath
+    catAttrs concatStrings concatStringsSep fileContents importJSON makeBinPath
     optional optionalString strings attrValues mapAttrs attrByPath
     traceValFn;
 
+  envFlags = callPackage ../tools/envParser.nix { };
+  enableNimbus = (attrByPath ["STATUS_GO_ENABLE_NIMBUS"] "0" envFlags) != "0";
   utils = callPackage ./utils.nix { inherit xcodeWrapper; };
   gomobile = callPackage ./gomobile { inherit (androidPkgs) platform-tools; inherit xcodeWrapper utils buildGoPackage; };
+  nimbus = if enableNimbus then callPackage ./nimbus { } else { wrappers-android = { }; };
   buildStatusGoDesktopLib = callPackage ./build-desktop-status-go.nix { inherit buildGoPackage go xcodeWrapper utils; };
-  buildStatusGoMobileLib = callPackage ./build-mobile-status-go.nix { inherit buildGoPackage go gomobile xcodeWrapper utils; };
+  buildStatusGoMobileLib = callPackage ./build-mobile-status-go.nix { inherit buildGoPackage go gomobile xcodeWrapper utils androidPkgs; };
   srcData =
     # If config.status-im.status-go.src-override is defined, instruct Nix to use that path to build status-go
     if (attrByPath ["status-im" "status-go" "src-override"] "" config) != "" then rec {
@@ -51,21 +54,43 @@ let
       };
 
   mobileConfigs = {
-    android = {
+    android = rec {
       name = "android";
-      outputFileName = "status-go-${srcData.shortRev}.aar";
       envVars = [
         "ANDROID_HOME=${androidPkgs.androidsdk}/libexec/android-sdk"
         "ANDROID_NDK_HOME=${androidPkgs.ndk-bundle}/libexec/android-sdk/ndk-bundle"
         "PATH=${makeBinPath [ openjdk ]}:$PATH"
       ];
-      gomobileExtraFlags = [ "-androidapi 18" ];
+      gomobileExtraFlags = [ "-androidapi 23" ];
+      outputFileName = "status-go-${srcData.shortRev}.aar";
+      platforms = {
+        arm64 = {
+          linkNimbus = enableNimbus;
+          nimbus = assert enableNimbus; nimbus.wrappers-android.arm64;
+          gomobileTarget = "${name}/arm64";
+          outputFileName = "status-go-${srcData.shortRev}-arm64.aar";
+        };
+        x86 = {
+          linkNimbus = enableNimbus;
+          nimbus = assert enableNimbus; nimbus.wrappers-android.x86;
+          gomobileTarget = "${name}/386";
+          outputFileName = "status-go-${srcData.shortRev}-386.aar";
+        };
+      };
     };
-    ios = {
+    ios = rec {
       name = "ios";
-      outputFileName = "Statusgo.framework";
       envVars = [];
       gomobileExtraFlags = [ "-iosversion=8.0" ];
+      outputFileName = "Statusgo.framework";
+      platforms = {
+        ios = {
+          linkNimbus = enableNimbus;
+          nimbus = assert false; null; # TODO: Currently we don't support Nimbus on iOS
+          gomobileTarget = name;
+          inherit outputFileName;
+        };
+      };
     };
   };
   hostConfigs = {
@@ -80,7 +105,7 @@ let
   };
   currentHostConfig = if stdenv.isDarwin then hostConfigs.darwin else hostConfigs.linux;
 
-  goBuildFlags = "-v";
+  goBuildFlags = concatStringsSep " " [ "-v" (optionalString enableNimbus "-tags='nimbus'") ];
   # status-go params to be set at build time, important for About section and metrics
   goBuildParams = {
     GitCommit = srcData.rev;

nix/status-go/nimbus/default.nix

@@ -0,0 +1,33 @@
+{ config, stdenv, callPackage, fetchFromGitHub, mkFilter }:
+
+let
+  inherit (stdenv.lib) attrByPath strings traceValFn;
+
+  repo = "nimbus";
+
+  localPath = attrByPath ["status-im" "nimbus" "src-override"] "" config;
+  localSrc = builtins.path rec { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control
+    path = traceValFn (path: "Using local ${repo} sources from ${path}\n") localPath;
+    name = "${repo}-source-local";
+    filter =
+      # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size
+      mkFilter {
+        dirRootsToInclude = [ "nix" "wrappers" "vendor" ];
+        dirsToExclude = [ ".git" ".svn" "CVS" ".hg" ".vscode" ".dependabot" ".github" "examples" "docs" ];
+        filesToInclude = [ "Makefile" "nim.cfg" "nimbus.nimble" "default.nix" ];
+        root = path;
+      };
+  };
+
+  src = if localPath != "" then localSrc
+        else fetchFromGitHub rec {
+          inherit repo;
+          name = "${repo}-source-${strings.substring 0 7 rev}";
+          rev = "501455b0cd2e74c451bc1743e2f1070a3fee1343";
+          owner = "status-im";
+          sha256 = "0nxh3hh8fib3hlmvs5d67h6cq3kyap94pa9w7ixsfa5285ila17h";
+          fetchSubmodules = true;
+        };
+  nimbusDeriv = import "${src}/nix/default.nix";
+
+in nimbusDeriv

scripts/release-android.sh

@@ -11,6 +11,9 @@ config+="status-im.build-type=\"${BUILD_TYPE}\";"
 if [ -n "${STATUS_GO_SRC_OVERRIDE}" ]; then
   config+="status-im.status-go.src-override=\"${STATUS_GO_SRC_OVERRIDE}\";"
 fi
+if [ -n "${NIMBUS_SRC_OVERRIDE}" ]; then
+  config+="status-im.nimbus.src-override=\"${NIMBUS_SRC_OVERRIDE}\";"
+fi
 config+="status-im.status-react.build-number=\"${BUILD_NUMBER}\";"
 config+="status-im.status-react.keystore-file=\"${STORE_FILE}\";"
 nixOpts=(

src/status_im/mailserver/core.cljs

@@ -286,7 +286,7 @@
   - mailserver disconnected: we try to reconnect
   - mailserver connected: we mark the mailserver as trusted peer"
   [{:keys [db] :as cofx} previous-summary]
-  (when (:multiaccount db)
+  (when (and (not config/nimbus-enabled?) (:multiaccount db))
     (if (:mailserver/current-id db)
       (let [{:keys [peers-summary peers-count]} db
             {:keys [address sym-key-id] :as mailserver} (fetch-current db)
@@ -303,8 +303,8 @@
           (mark-trusted-peer cofx)
           mailserver-removed?
           (connect-to-mailserver cofx)))
-      ;; if there is no current mailserver defined,
+        ;; if there is no current mailserver defined,
-      ;; we set it first
+        ;; we set it first
       (set-current-mailserver cofx))))
 
 (defn adjust-request-for-transit-time

src/status_im/subs.cljs

@@ -320,7 +320,7 @@
  :disconnected?
  :<- [:peers-count]
  (fn [peers-count]
-   (zero? peers-count)))
+   (and (not config/nimbus-enabled?) (zero? peers-count))))
 
 (re-frame/reg-sub
  :offline?

src/status_im/transport/core.cljs

@@ -42,7 +42,7 @@
             (fetch-node-info-fx)
             (pairing/init)
             (publisher/start-fx)
-            (mailserver/initialize-mailserver)))
+            (when-not config/nimbus-enabled? (mailserver/initialize-mailserver))))
 
 (fx/defn stop-whisper
   "Stops whisper protocol"

src/status_im/utils/config.cljs

@@ -50,3 +50,4 @@
 (def pow-target (js/parseFloat (get-config :POW_TARGET "0.002")))
 (def pow-time (js/parseInt (get-config :POW_TIME "1")))
 (def max-installations 2)
+(def nimbus-enabled? (get-config :STATUS_GO_ENABLE_NIMBUS false))