From 2dce0ba861c66ed0176dd7899b366dc21ebf9420 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Tue, 21 Jan 2020 08:19:49 +0100 Subject: [PATCH] Integrate status-go Nimbus build Signed-off-by: Pedro Pombeiro --- .env | 1 + .env.jenkins | 1 + Makefile | 2 +- nix/README.md | 5 + .../android/targets/release-android.nix | 2 + nix/scripts/shell.sh | 3 + nix/status-go/build-mobile-status-go.nix | 109 ++++++++++++------ nix/status-go/build-status-go.nix | 2 + nix/status-go/default.nix | 41 +++++-- nix/status-go/nimbus/default.nix | 33 ++++++ scripts/release-android.sh | 3 + src/status_im/mailserver/core.cljs | 6 +- src/status_im/subs.cljs | 2 +- src/status_im/transport/core.cljs | 2 +- src/status_im/utils/config.cljs | 1 + 15 files changed, 164 insertions(+), 49 deletions(-) create mode 100644 nix/status-go/nimbus/default.nix diff --git a/.env b/.env index 61e82ebd25..2d8d4e1d17 100644 --- a/.env +++ b/.env @@ -21,3 +21,4 @@ PARTITIONED_TOPIC=0 CONTRACT_NODES=1 MOBILE_UI_FOR_DESKTOP=1 STATUS_GO_PROTOCOL=0 +STATUS_GO_ENABLE_NIMBUS=0 diff --git a/.env.jenkins b/.env.jenkins index cc23ea3516..0dbea682b4 100644 --- a/.env.jenkins +++ b/.env.jenkins @@ -20,3 +20,4 @@ RPC_NETWORKS_ONLY=0 PARTITIONED_TOPIC=0 CONTRACT_NODES=1 MOBILE_UI_FOR_DESKTOP=1 +STATUS_GO_ENABLE_NIMBUS=0 diff --git a/Makefile b/Makefile index 5db529cc60..dc489141be 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ export REACT_SERVER_PORT ?= 5001 # Our custom config is located in nix/nix.conf export NIX_CONF_DIR = $(PWD)/nix # Defines which variables will be kept for Nix pure shell, use semicolon as divider -export _NIX_KEEP ?= TMPDIR,BUILD_ENV,STATUS_GO_SRC_OVERRIDE +export _NIX_KEEP ?= TMPDIR,BUILD_ENV,STATUS_GO_SRC_OVERRIDE,NIMBUS_SRC_OVERRIDE export _NIX_ROOT = /nix # legacy TARGET_OS variable support ifdef TARGET_OS diff --git a/nix/README.md b/nix/README.md index ec3952f4d7..51fa98dfba 100644 --- a/nix/README.md +++ b/nix/README.md @@ -62,6 +62,11 @@ or for a one-off build: make release-android STATUS_GO_SRC_OVERRIDE=$GOPATH/src/github.com/status-im/status-go ``` +## Using a local Nimbus repository + +If you need to use a locally checked-out Nimbus repository as a dependency of status-go, you can achieve that by defining the `NIMBUS_SRC_OVERRIDE` +environment variable, in the same way as the previous point for local status-go repositories. + ## Known Issues ### MacOS 10.15 "Catalina" diff --git a/nix/mobile/android/targets/release-android.nix b/nix/mobile/android/targets/release-android.nix index 9b8bf3a272..ad78412ef1 100644 --- a/nix/mobile/android/targets/release-android.nix +++ b/nix/mobile/android/targets/release-android.nix @@ -14,6 +14,8 @@ let inherit (lib) attrByPath hasAttrByPath optionalAttrs; env' = env // optionalAttrs (hasAttrByPath ["status-im" "status-go" "src-override"] config) { STATUS_GO_SRC_OVERRIDE = config.status-im.status-go.src-override; + } // optionalAttrs (hasAttrByPath ["status-im" "nimbus" "src-override"] config) { + NIMBUS_SRC_OVERRIDE = config.status-im.nimbus.src-override; }; inherit (config.status-im) build-type; inherit (config.status-im.status-react) build-number; diff --git a/nix/scripts/shell.sh b/nix/scripts/shell.sh index 9821ab98c7..6faab02216 100755 --- a/nix/scripts/shell.sh +++ b/nix/scripts/shell.sh @@ -55,6 +55,9 @@ config='' if [ -n "${STATUS_GO_SRC_OVERRIDE}" ]; then config+="status-im.status-go.src-override=\"${STATUS_GO_SRC_OVERRIDE}\";" fi +if [ -n "${NIMBUS_SRC_OVERRIDE}" ]; then + config+="status-im.nimbus.src-override=\"${NIMBUS_SRC_OVERRIDE}\";" +fi config+="status-im.build-type=\"${BUILD_TYPE}\";" if [ -n "$config" ]; then diff --git a/nix/status-go/build-mobile-status-go.nix b/nix/status-go/build-mobile-status-go.nix index da58e06a7a..5f5f6bef5a 100644 --- a/nix/status-go/build-mobile-status-go.nix +++ b/nix/status-go/build-mobile-status-go.nix @@ -1,5 +1,5 @@ { stdenv, utils, callPackage, - buildGoPackage, go, gomobile, openjdk, xcodeWrapper }: + buildGoPackage, go, gomobile, androidPkgs, openjdk, unzip, zip, xcodeWrapper }: { owner, repo, rev, cleanVersion, goPackagePath, src, host, @@ -8,7 +8,7 @@ targetConfig } @ args': let - inherit (stdenv.lib) concatStringsSep makeBinPath optional; + inherit (stdenv.lib) concatStringsSep makeBinPath optional optionals; buildStatusGo = callPackage ./build-status-go.nix { inherit buildGoPackage go xcodeWrapper utils; @@ -19,44 +19,83 @@ let "targetConfig" "goBuildFlags" "goBuildLdFlags" ]; - buildStatusGoMobileLib = buildStatusGo (args // { - nativeBuildInputs = [ gomobile ] ++ optional (targetConfig.name == "android") openjdk; + buildStatusGoMobileLib = + let + inherit (stdenv.lib) concatStrings mapAttrsToList optionalString; + in buildStatusGo (args // { + nativeBuildInputs = [ gomobile unzip zip ] ++ optional (targetConfig.name == "android") openjdk; - buildMessage = "Building mobile library for ${targetConfig.name}"; - # Build mobile libraries - buildPhase = - let - NIX_GOWORKDIR = "$NIX_BUILD_TOP/go-build"; - CGO_LDFLAGS = concatStringsSep " " goBuildLdFlags; - in with targetConfig; '' - mkdir ${NIX_GOWORKDIR} + buildMessage = "Building mobile library for ${targetConfig.name}"; + # Build mobile libraries + buildPhase = + let + NIX_GOWORKDIR = "$NIX_BUILD_TOP/go-build"; + CGO_LDFLAGS = concatStringsSep " " (goBuildLdFlags ++ [ "-extldflags=-Wl,--allow-multiple-definition" ]); + nimbusBridgeVendorDir = "$NIX_BUILD_TOP/go/src/${goPackagePath}/vendor/${goPackagePath}/eth-node/bridge/nimbus"; + in '' + mkdir ${NIX_GOWORKDIR} - export GO111MODULE=off - export GOPATH=${gomobile.dev}:$GOPATH - export PATH=${makeBinPath [ gomobile.bin ]}:$PATH - export NIX_GOWORKDIR=${NIX_GOWORKDIR} - export ${concatStringsSep " " envVars} - gomobile bind \ - -target=${name} \ - -ldflags='${CGO_LDFLAGS}' \ - ${concatStringsSep " " gomobileExtraFlags} \ - ${goBuildFlags} \ - -o ${outputFileName} \ - ${goPackagePath}/mobile + export GO111MODULE=off + export GOPATH=${gomobile.dev}:$GOPATH + export PATH=${makeBinPath [ gomobile.bin ]}:$PATH + export NIX_GOWORKDIR=${NIX_GOWORKDIR} + export ${concatStringsSep " " targetConfig.envVars} - rm -rf ${NIX_GOWORKDIR} - ''; + # Build the Go library using gomobile for each of the configured platforms + ${concatStrings (mapAttrsToList (_: platformConfig: '' - installPhase = '' - mkdir -p $out/lib - mv ${targetConfig.outputFileName} $out/lib/ - ''; + ${optionalString platformConfig.linkNimbus '' + # Copy the Nimbus API artifacts to the expected vendor location + cp ${platformConfig.nimbus}/{include/*,lib/libnimbus.a} ${nimbusBridgeVendorDir} + chmod +w ${nimbusBridgeVendorDir}/libnimbus.{a,h} + ''} - outputs = [ "out" ]; + echo + echo "Building for target ${platformConfig.gomobileTarget}" + gomobile bind \ + -target=${platformConfig.gomobileTarget} \ + -ldflags="${CGO_LDFLAGS}" \ + ${concatStringsSep " " targetConfig.gomobileExtraFlags} \ + ${goBuildFlags} \ + -o ${platformConfig.outputFileName} \ + ${goPackagePath}/mobile - meta = { - platforms = with stdenv.lib.platforms; linux ++ darwin; - }; - }); + ${optionalString platformConfig.linkNimbus '' + rm ${nimbusBridgeVendorDir}/libnimbus.{a,h} + ''} + '') targetConfig.platforms) + } + + if [ "${targetConfig.name}" = 'android' ]; then + # Merge the platform-specific .aar files into a single one + local mergeDir='.aar' + mkdir $mergeDir + ${concatStrings (mapAttrsToList (_: platformConfig: '' + unzip -d $mergeDir -q -n -u ${platformConfig.outputFileName} + rm ${platformConfig.outputFileName} + '') targetConfig.platforms)} + pushd $mergeDir > /dev/null + zip -r -o ../${targetConfig.outputFileName} * + popd > /dev/null + rm -rf $mergeDir + unzip -l ${targetConfig.outputFileName} + fi + + # TODO: Merge iOS packages when linking with libnimbus.a + + rm -rf ${NIX_GOWORKDIR} + ''; + + installPhase = '' + mkdir -p $out/lib + mv ${targetConfig.outputFileName} $out/lib/ + ''; + + outputs = [ "out" ]; + + meta = { + platforms = with stdenv.lib.platforms; linux ++ darwin; + }; + }); in buildStatusGoMobileLib diff --git a/nix/status-go/build-status-go.nix b/nix/status-go/build-status-go.nix index 3380474b38..3390707e82 100644 --- a/nix/status-go/build-status-go.nix +++ b/nix/status-go/build-status-go.nix @@ -2,6 +2,7 @@ { owner, repo, rev, cleanVersion, goPackagePath, src, host, nativeBuildInputs ? [], + buildInputs ? [], buildPhase, buildMessage, installPhase ? "", postInstall ? "", @@ -24,6 +25,7 @@ let nativeBuildInputs = nativeBuildInputs ++ lib.optional isDarwin xcodeWrapper; + inherit buildInputs; # Fixes Cgo related build failures (see https://github.com/NixOS/nixpkgs/issues/25959 ) hardeningDisable = [ "fortify" ]; diff --git a/nix/status-go/default.nix b/nix/status-go/default.nix index c9d2e170c1..3fcdc881e1 100644 --- a/nix/status-go/default.nix +++ b/nix/status-go/default.nix @@ -3,14 +3,17 @@ let inherit (stdenv.lib) - catAttrs concatStrings fileContents importJSON makeBinPath + catAttrs concatStrings concatStringsSep fileContents importJSON makeBinPath optional optionalString strings attrValues mapAttrs attrByPath traceValFn; + envFlags = callPackage ../tools/envParser.nix { }; + enableNimbus = (attrByPath ["STATUS_GO_ENABLE_NIMBUS"] "0" envFlags) != "0"; utils = callPackage ./utils.nix { inherit xcodeWrapper; }; gomobile = callPackage ./gomobile { inherit (androidPkgs) platform-tools; inherit xcodeWrapper utils buildGoPackage; }; + nimbus = if enableNimbus then callPackage ./nimbus { } else { wrappers-android = { }; }; buildStatusGoDesktopLib = callPackage ./build-desktop-status-go.nix { inherit buildGoPackage go xcodeWrapper utils; }; - buildStatusGoMobileLib = callPackage ./build-mobile-status-go.nix { inherit buildGoPackage go gomobile xcodeWrapper utils; }; + buildStatusGoMobileLib = callPackage ./build-mobile-status-go.nix { inherit buildGoPackage go gomobile xcodeWrapper utils androidPkgs; }; srcData = # If config.status-im.status-go.src-override is defined, instruct Nix to use that path to build status-go if (attrByPath ["status-im" "status-go" "src-override"] "" config) != "" then rec { @@ -51,21 +54,43 @@ let }; mobileConfigs = { - android = { + android = rec { name = "android"; - outputFileName = "status-go-${srcData.shortRev}.aar"; envVars = [ "ANDROID_HOME=${androidPkgs.androidsdk}/libexec/android-sdk" "ANDROID_NDK_HOME=${androidPkgs.ndk-bundle}/libexec/android-sdk/ndk-bundle" "PATH=${makeBinPath [ openjdk ]}:$PATH" ]; - gomobileExtraFlags = [ "-androidapi 18" ]; + gomobileExtraFlags = [ "-androidapi 23" ]; + outputFileName = "status-go-${srcData.shortRev}.aar"; + platforms = { + arm64 = { + linkNimbus = enableNimbus; + nimbus = assert enableNimbus; nimbus.wrappers-android.arm64; + gomobileTarget = "${name}/arm64"; + outputFileName = "status-go-${srcData.shortRev}-arm64.aar"; + }; + x86 = { + linkNimbus = enableNimbus; + nimbus = assert enableNimbus; nimbus.wrappers-android.x86; + gomobileTarget = "${name}/386"; + outputFileName = "status-go-${srcData.shortRev}-386.aar"; + }; + }; }; - ios = { + ios = rec { name = "ios"; - outputFileName = "Statusgo.framework"; envVars = []; gomobileExtraFlags = [ "-iosversion=8.0" ]; + outputFileName = "Statusgo.framework"; + platforms = { + ios = { + linkNimbus = enableNimbus; + nimbus = assert false; null; # TODO: Currently we don't support Nimbus on iOS + gomobileTarget = name; + inherit outputFileName; + }; + }; }; }; hostConfigs = { @@ -80,7 +105,7 @@ let }; currentHostConfig = if stdenv.isDarwin then hostConfigs.darwin else hostConfigs.linux; - goBuildFlags = "-v"; + goBuildFlags = concatStringsSep " " [ "-v" (optionalString enableNimbus "-tags='nimbus'") ]; # status-go params to be set at build time, important for About section and metrics goBuildParams = { GitCommit = srcData.rev; diff --git a/nix/status-go/nimbus/default.nix b/nix/status-go/nimbus/default.nix new file mode 100644 index 0000000000..6bb37df3dc --- /dev/null +++ b/nix/status-go/nimbus/default.nix @@ -0,0 +1,33 @@ +{ config, stdenv, callPackage, fetchFromGitHub, mkFilter }: + +let + inherit (stdenv.lib) attrByPath strings traceValFn; + + repo = "nimbus"; + + localPath = attrByPath ["status-im" "nimbus" "src-override"] "" config; + localSrc = builtins.path rec { # We use builtins.path so that we can name the resulting derivation, otherwise the name would be taken from the checkout directory, which is outside of our control + path = traceValFn (path: "Using local ${repo} sources from ${path}\n") localPath; + name = "${repo}-source-local"; + filter = + # Keep this filter as restrictive as possible in order to avoid unnecessary rebuilds and limit closure size + mkFilter { + dirRootsToInclude = [ "nix" "wrappers" "vendor" ]; + dirsToExclude = [ ".git" ".svn" "CVS" ".hg" ".vscode" ".dependabot" ".github" "examples" "docs" ]; + filesToInclude = [ "Makefile" "nim.cfg" "nimbus.nimble" "default.nix" ]; + root = path; + }; + }; + + src = if localPath != "" then localSrc + else fetchFromGitHub rec { + inherit repo; + name = "${repo}-source-${strings.substring 0 7 rev}"; + rev = "501455b0cd2e74c451bc1743e2f1070a3fee1343"; + owner = "status-im"; + sha256 = "0nxh3hh8fib3hlmvs5d67h6cq3kyap94pa9w7ixsfa5285ila17h"; + fetchSubmodules = true; + }; + nimbusDeriv = import "${src}/nix/default.nix"; + +in nimbusDeriv diff --git a/scripts/release-android.sh b/scripts/release-android.sh index 3796bccd13..a1fddae950 100755 --- a/scripts/release-android.sh +++ b/scripts/release-android.sh @@ -11,6 +11,9 @@ config+="status-im.build-type=\"${BUILD_TYPE}\";" if [ -n "${STATUS_GO_SRC_OVERRIDE}" ]; then config+="status-im.status-go.src-override=\"${STATUS_GO_SRC_OVERRIDE}\";" fi +if [ -n "${NIMBUS_SRC_OVERRIDE}" ]; then + config+="status-im.nimbus.src-override=\"${NIMBUS_SRC_OVERRIDE}\";" +fi config+="status-im.status-react.build-number=\"${BUILD_NUMBER}\";" config+="status-im.status-react.keystore-file=\"${STORE_FILE}\";" nixOpts=( diff --git a/src/status_im/mailserver/core.cljs b/src/status_im/mailserver/core.cljs index 7ae7f6b83b..d4a573c411 100644 --- a/src/status_im/mailserver/core.cljs +++ b/src/status_im/mailserver/core.cljs @@ -286,7 +286,7 @@ - mailserver disconnected: we try to reconnect - mailserver connected: we mark the mailserver as trusted peer" [{:keys [db] :as cofx} previous-summary] - (when (:multiaccount db) + (when (and (not config/nimbus-enabled?) (:multiaccount db)) (if (:mailserver/current-id db) (let [{:keys [peers-summary peers-count]} db {:keys [address sym-key-id] :as mailserver} (fetch-current db) @@ -303,8 +303,8 @@ (mark-trusted-peer cofx) mailserver-removed? (connect-to-mailserver cofx))) - ;; if there is no current mailserver defined, - ;; we set it first + ;; if there is no current mailserver defined, + ;; we set it first (set-current-mailserver cofx)))) (defn adjust-request-for-transit-time diff --git a/src/status_im/subs.cljs b/src/status_im/subs.cljs index 310be45322..759e191245 100644 --- a/src/status_im/subs.cljs +++ b/src/status_im/subs.cljs @@ -320,7 +320,7 @@ :disconnected? :<- [:peers-count] (fn [peers-count] - (zero? peers-count))) + (and (not config/nimbus-enabled?) (zero? peers-count)))) (re-frame/reg-sub :offline? diff --git a/src/status_im/transport/core.cljs b/src/status_im/transport/core.cljs index d622659928..064b0120de 100644 --- a/src/status_im/transport/core.cljs +++ b/src/status_im/transport/core.cljs @@ -42,7 +42,7 @@ (fetch-node-info-fx) (pairing/init) (publisher/start-fx) - (mailserver/initialize-mailserver))) + (when-not config/nimbus-enabled? (mailserver/initialize-mailserver)))) (fx/defn stop-whisper "Stops whisper protocol" diff --git a/src/status_im/utils/config.cljs b/src/status_im/utils/config.cljs index cbed3daf1f..1fe3c50e17 100644 --- a/src/status_im/utils/config.cljs +++ b/src/status_im/utils/config.cljs @@ -50,3 +50,4 @@ (def pow-target (js/parseFloat (get-config :POW_TARGET "0.002"))) (def pow-time (js/parseInt (get-config :POW_TIME "1"))) (def max-installations 2) +(def nimbus-enabled? (get-config :STATUS_GO_ENABLE_NIMBUS false))