From aeacbd928e13a50def46cce5137be219ba3ceff2 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro Date: Wed, 3 Apr 2019 13:06:42 +0200 Subject: [PATCH] Separate logic into shell.nix and derivation.nix to fix `nix build` command Signed-off-by: Pedro Pombeiro --- ci/Jenkinsfile.android | 2 +- ci/Jenkinsfile.linux | 2 +- ci/Jenkinsfile.nix.linux | 11 +-- ci/Jenkinsfile.nix.macos | 5 +- ci/Jenkinsfile.windows | 2 +- ci/docker/nix/Dockerfile | 3 +- ci/utils.groovy | 6 +- default.nix | 112 ++++------------------------- derivation.nix | 87 ++++++++++++++++++++++ nix/desktop/default.nix | 14 ++-- nix/desktop/windows/default.nix | 2 +- nix/mobile/default.nix | 9 ++- nix/shell.sh | 4 +- nix/status-go/gomobile/default.nix | 8 +-- scripts/run-environment-check.sh | 2 +- shell.nix | 41 +++++++++++ 16 files changed, 179 insertions(+), 131 deletions(-) create mode 100644 derivation.nix create mode 100644 shell.nix diff --git a/ci/Jenkinsfile.android b/ci/Jenkinsfile.android index 2b1444be99..2952858bc0 100644 --- a/ci/Jenkinsfile.android +++ b/ci/Jenkinsfile.android @@ -2,7 +2,7 @@ pipeline { agent { docker { label 'linux' - image 'statusteam/nix:jenkins-1.0.0-150d5721' + image 'statusteam/nix:jenkins-1.0.0-e20e5e15' args ( "-v /home/jenkins/tmp:/var/tmp:rw "+ "-v /home/jenkins/status-im.keystore:/tmp/status-im.keystore:ro" diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux index 77a5117647..ed8c93eee3 100644 --- a/ci/Jenkinsfile.linux +++ b/ci/Jenkinsfile.linux @@ -2,7 +2,7 @@ pipeline { agent { docker { label 'linux' - image 'statusteam/nix:jenkins-1.0.0-150d5721' + image 'statusteam/nix:jenkins-1.0.0-e20e5e15' args ( "-v /tmp/Android/Sdk:/home/jenkins/.status/Android/Sdk:rw "+ "-v /var/tmp/lein:/var/tmp/lein:rw "+ diff --git a/ci/Jenkinsfile.nix.linux b/ci/Jenkinsfile.nix.linux index bd2fbf2bf8..7d0f22f735 100644 --- a/ci/Jenkinsfile.nix.linux +++ b/ci/Jenkinsfile.nix.linux @@ -3,18 +3,18 @@ pipeline { /* the -u is necessary for acces to /nix */ docker { label 'linux' - image 'statusteam/nix:jenkins-1.0.0-150d5721' + image 'statusteam/nix:jenkins-1.0.0-e20e5e15' } } environment { + CI_ENVIRONMENT = 'jenkins' /* we source .bash_profile to be able to use nix-store */ NIX_SSHOPTS = "-o StrictHostKeyChecking=no source .bash_profile;" /* where our /nix/store is hosted */ NIX_CACHE_USER = 'nix-cache' NIX_CACHE_HOST = 'master-01.do-ams3.ci.misc.statusim.net' - /* we add both keys so default binary cache also works */ - NIX_CONF_DIR = "${env.WORKSPACE}/scripts/lib/setup/nix" + NIX_CONF_DIR = "${env.WORKSPACE}/nix" } options { @@ -37,7 +37,10 @@ pipeline { } stage('Build') { steps { - sh 'nix-shell --run echo' + sh """ + nix build --no-link && + nix-shell --run echo + """ } } stage('Upload') { diff --git a/ci/Jenkinsfile.nix.macos b/ci/Jenkinsfile.nix.macos index 00f5c6eecb..766d0d8a78 100644 --- a/ci/Jenkinsfile.nix.macos +++ b/ci/Jenkinsfile.nix.macos @@ -4,13 +4,13 @@ pipeline { } environment { + CI_ENVIRONMENT = 'jenkins' /* we source .bash_profile to be able to use nix-store */ NIX_SSHOPTS = "-o StrictHostKeyChecking=no source .bash_profile;" /* where our /nix/store is hosted */ NIX_CACHE_USER = 'nix-cache' NIX_CACHE_HOST = 'master-01.do-ams3.ci.misc.statusim.net' - /* we add both keys so default binary cache also works */ - NIX_CONF_DIR = "${env.WORKSPACE}/scripts/lib/setup/nix" + NIX_CONF_DIR = "${env.WORKSPACE}/nix" } options { @@ -39,6 +39,7 @@ pipeline { steps { sh """ . ~/.nix-profile/etc/profile.d/nix.sh && \ + nix build --no-link && \ nix-shell --run echo """ } diff --git a/ci/Jenkinsfile.windows b/ci/Jenkinsfile.windows index 9a04f4d358..248d5add6f 100644 --- a/ci/Jenkinsfile.windows +++ b/ci/Jenkinsfile.windows @@ -2,7 +2,7 @@ pipeline { agent { docker { label 'linux-new' - image 'statusteam/nix:jenkins-1.0.0-150d5721' + image 'statusteam/nix:jenkins-1.0.0-e20e5e15' args ( "-v /var/tmp/lein:/var/tmp/lein:rw "+ "-v /var/tmp/npm:/var/tmp/npm:rw " diff --git a/ci/docker/nix/Dockerfile b/ci/docker/nix/Dockerfile index 27dec1d4fc..75a68f4918 100644 --- a/ci/docker/nix/Dockerfile +++ b/ci/docker/nix/Dockerfile @@ -87,7 +87,8 @@ ARG GIT_COMMIT RUN export USER=jenkins \ && /tmp/nix-install.sh \ && . ~/.nix-profile/etc/profile.d/nix.sh \ - && nix-build --no-out-link https://github.com/status-im/status-react/tarball/${GIT_COMMIT} + && nix-build --no-out-link https://github.com/status-im/status-react/tarball/${GIT_COMMIT} \ + && nix-shell --run 'echo' https://github.com/status-im/status-react/tarball/${GIT_COMMIT} # hack to avoid calling login ENV USER=jenkins diff --git a/ci/utils.groovy b/ci/utils.groovy index 31424ea665..3e81cfc749 100644 --- a/ci/utils.groovy +++ b/ci/utils.groovy @@ -20,9 +20,9 @@ def nix_sh(cmd) { set +x . ~/.nix-profile/etc/profile.d/nix.sh set -x - nix-shell \'${env.WORKSPACE}/default.nix\' \\ - --argstr target-os \'${env.TARGET_PLATFORM}\' \\ - --run \'${cmd}\' + nix-shell --argstr target-os \'${env.TARGET_PLATFORM}\' \\ + --run \'${cmd}\' \\ + \'${env.WORKSPACE}/shell.nix\' """ } diff --git a/default.nix b/default.nix index d0015e1796..9bd4898514 100644 --- a/default.nix +++ b/default.nix @@ -1,103 +1,19 @@ -# target-os = [ 'windows' 'linux' 'macos' 'android' 'ios' ] -{ pkgs ? import ((import { }).fetchFromGitHub { +# target-os = [ 'windows' 'linux' 'macos' 'android' 'ios' 'all' ] +{ config ? { android_sdk.accept_license = true; }, + pkgs ? import ((import { }).fetchFromGitHub { owner = "status-im"; repo = "nixpkgs"; rev = "db492b61572251c2866f6b5e6e94e9d70e7d3021"; sha256 = "188r7gbcrxi20nj6xh9bmdf3lbjwb94v9s0wpacl7q39g1fca66h"; - }) { config = { android_sdk.accept_license = true; }; }, - target-os ? "" }: + }) { inherit config; }, + stdenv ? pkgs.stdenv, + target-os ? "all" }: -with pkgs; - let - targetDesktop = { - "linux" = true; - "windows" = true; - "macos" = true; - "" = true; - }.${target-os} or false; - targetMobile = { - "android" = true; - "ios" = true; - "" = true; - }.${target-os} or false; - # TODO: Try to use stdenv for iOS. The problem is with building iOS as the build is trying to pass parameters to Apple's ld that are meant for GNU's ld (e.g. -dynamiclib) - _stdenv = if target-os == "ios" || target-os == "" then stdenvNoCC else stdenv; - statusDesktop = callPackage ./nix/desktop { inherit target-os; stdenv = _stdenv; }; - statusMobile = callPackage ./nix/mobile { inherit target-os status-go; androidPkgs = androidComposition; stdenv = _stdenv; }; - status-go = callPackage ./nix/status-go { inherit (xcodeenv) composeXcodeWrapper; inherit xcodewrapperArgs; androidPkgs = androidComposition; }; - nodeInputs = import ./nix/global-node-packages/output { - # The remaining dependencies come from Nixpkgs - inherit pkgs nodejs; - }; - nodePkgs = [ - nodejs - python27 # for e.g. gyp - yarn - ] ++ (map (x: nodeInputs."${x}") (builtins.attrNames nodeInputs)); - xcodewrapperArgs = { - version = "10.1"; - }; - xcodeWrapper = xcodeenv.composeXcodeWrapper xcodewrapperArgs; - androidComposition = androidenv.composeAndroidPackages { - toolsVersion = "26.1.1"; - platformToolsVersion = "28.0.2"; - buildToolsVersions = [ "28.0.3" ]; - includeEmulator = false; - platformVersions = [ "26" "27" ]; - includeSources = false; - includeDocs = false; - includeSystemImages = false; - systemImageTypes = [ "default" ]; - abiVersions = [ "armeabi-v7a" ]; - lldbVersions = [ "2.0.2558144" ]; - cmakeVersions = [ "3.6.4111459" ]; - includeNDK = true; - ndkVersion = "19.2.5345600"; - useGoogleAPIs = false; - useGoogleTVAddOns = false; - includeExtras = [ "extras;android;m2repository" "extras;google;m2repository" ]; - }; - - in _stdenv.mkDerivation rec { - name = "env"; - env = buildEnv { name = name; paths = buildInputs; }; - buildInputs = with _stdenv; [ - bash - clojure - curl - git - jq - leiningen - lsof # used in scripts/start-react-native.sh - maven - ncurses - ps # used in scripts/start-react-native.sh - watchman - unzip - wget - - status-go - ] ++ nodePkgs - ++ lib.optional isDarwin cocoapods - ++ lib.optional targetDesktop statusDesktop.buildInputs - ++ lib.optional targetMobile statusMobile.buildInputs; - shellHook = - '' - set -e - '' + - status-go.shellHook + - '' - export STATUS_GO_INCLUDEDIR=${status-go}/include - export STATUS_GO_LIBDIR=${status-go}/lib - export STATUS_GO_BINDIR=${status-go.bin}/bin - '' + - lib.optionalString targetDesktop statusDesktop.shellHook + - lib.optionalString targetMobile statusMobile.shellHook + - '' - if [ -n "$ANDROID_SDK_ROOT" ] && [ ! -d "$ANDROID_SDK_ROOT" ]; then - ./scripts/setup # we assume that if the Android SDK dir does not exist, setup script needs to be run - fi - set +e - ''; - hardeningDisable = status-go.hardeningDisable; - } +(pkgs.callPackage ./derivation.nix { inherit pkgs target-os config; }).overrideAttrs(_: { + src = null; + # TODO: Figure out if there's a better way to do this + # NOTE: There's a weird difference in behavior between Linux and macOS: in Linux the packages will only be fetched by `nix build` if unpackPhase exists. In macOS it's the other way around :-/ + phases = with stdenv; lib.optional isLinux [ "unpackPhase" ] ++ [ "noPhase" ]; + unpackPhase = "true"; + noPhase = "mkdir -p $out"; +}) diff --git a/derivation.nix b/derivation.nix new file mode 100644 index 0000000000..83a834f0cf --- /dev/null +++ b/derivation.nix @@ -0,0 +1,87 @@ +# target-os = [ 'windows' 'linux' 'macos' 'android' 'ios' ] +{ system ? builtins.currentSystem +, config ? {}, overlays ? [] +, pkgs ? (import { inherit system config overlays; }) +, target-os }: + +with pkgs; + let + sanitized-target-os = + if (builtins.any (os: target-os == os) [ "linux" "android" "windows" "macos" "ios" "all" ]) + then target-os + else throw "Unknown value for target-os: '${target-os}'"; + targetDesktop = { + "linux" = true; + "windows" = true; + "macos" = true; + "all" = true; + }.${sanitized-target-os} or false; + targetMobile = { + "android" = true; + "ios" = true; + "all" = true; + }.${sanitized-target-os} or false; + # TODO: Try to use stdenv for iOS. The problem is with building iOS as the build is trying to pass parameters to Apple's ld that are meant for GNU's ld (e.g. -dynamiclib) + _stdenv = stdenvNoCC; + statusDesktop = callPackage ./nix/desktop { inherit target-os; stdenv = _stdenv; }; + statusMobile = callPackage ./nix/mobile { inherit target-os status-go; androidPkgs = androidComposition; stdenv = _stdenv; }; + status-go = callPackage ./nix/status-go { inherit (xcodeenv) composeXcodeWrapper; inherit xcodewrapperArgs; androidPkgs = androidComposition; }; + nodeInputs = import ./nix/global-node-packages/output { + # The remaining dependencies come from Nixpkgs + inherit pkgs nodejs; + }; + nodePkgs = [ + nodejs + python27 # for e.g. gyp + yarn + ] ++ (map (x: nodeInputs."${x}") (builtins.attrNames nodeInputs)); + xcodewrapperArgs = { + version = "10.1"; + }; + xcodeWrapper = xcodeenv.composeXcodeWrapper xcodewrapperArgs; + androidComposition = androidenv.composeAndroidPackages { + toolsVersion = "26.1.1"; + platformToolsVersion = "28.0.2"; + buildToolsVersions = [ "28.0.3" ]; + includeEmulator = false; + platformVersions = [ "26" "27" ]; + includeSources = false; + includeDocs = false; + includeSystemImages = false; + systemImageTypes = [ "default" ]; + abiVersions = [ "armeabi-v7a" ]; + lldbVersions = [ "2.0.2558144" ]; + cmakeVersions = [ "3.6.4111459" ]; + includeNDK = true; + ndkVersion = "19.2.5345600"; + useGoogleAPIs = false; + useGoogleTVAddOns = false; + includeExtras = [ "extras;android;m2repository" "extras;google;m2repository" ]; + }; + + in _stdenv.mkDerivation rec { + name = "status-react-build-env"; + + buildInputs = with _stdenv; [ + clojure + leiningen + maven + watchman + + status-go + ] ++ nodePkgs + ++ lib.optional isDarwin cocoapods + ++ lib.optional (!isDarwin) gcc7 + ++ lib.optionals targetDesktop statusDesktop.buildInputs + ++ lib.optionals targetMobile statusMobile.buildInputs; + shellHook = + status-go.shellHook + + '' + export STATUS_GO_INCLUDEDIR=${status-go}/include + export STATUS_GO_LIBDIR=${status-go}/lib + export STATUS_GO_BINDIR=${status-go.bin}/bin + '' + + lib.optionalString targetDesktop statusDesktop.shellHook + + lib.optionalString targetMobile statusMobile.shellHook; + hardeningDisable = status-go.hardeningDisable; + } diff --git a/nix/desktop/default.nix b/nix/desktop/default.nix index 0417f252a3..3cbd4791ce 100644 --- a/nix/desktop/default.nix +++ b/nix/desktop/default.nix @@ -6,15 +6,15 @@ with stdenv; let targetLinux = { "linux" = true; - "" = isLinux; + "all" = isLinux; }.${target-os} or false; targetDarwin = { "macos" = true; - "" = isDarwin; + "all" = isDarwin; }.${target-os} or false; targetWindows = { "windows" = true; - "" = isLinux; + "all" = isLinux; }.${target-os} or false; linuxPlatform = callPackage ./linux { }; darwinPlatform = callPackage ./macos { }; @@ -26,10 +26,10 @@ in cmake extra-cmake-modules file - ] ++ lib.optional targetLinux linuxPlatform.buildInputs - ++ lib.optional targetDarwin darwinPlatform.buildInputs - ++ lib.optional (! targetWindows) qt5.full - ++ lib.optional targetWindows windowsPlatform.buildInputs; + ] ++ lib.optionals targetLinux linuxPlatform.buildInputs + ++ lib.optionals targetDarwin darwinPlatform.buildInputs + ++ lib.optionals targetWindows windowsPlatform.buildInputs + ++ lib.optional (! targetWindows) qt5.full; shellHook = (if target-os == "windows" then '' unset QT_PATH '' else '' diff --git a/nix/desktop/windows/default.nix b/nix/desktop/windows/default.nix index a50bc1914e..6c96ce73b9 100644 --- a/nix/desktop/windows/default.nix +++ b/nix/desktop/windows/default.nix @@ -8,7 +8,7 @@ let in { - buildInputs = lib.optional isLinux [ + buildInputs = lib.optionals isLinux [ conan nsis baseImage diff --git a/nix/mobile/default.nix b/nix/mobile/default.nix index a2ff637032..cbcbe57af0 100644 --- a/nix/mobile/default.nix +++ b/nix/mobile/default.nix @@ -1,4 +1,4 @@ -{ stdenv, pkgs, target-os ? "", status-go, androidPkgs }: +{ stdenv, pkgs, target-os ? "all", status-go, androidPkgs }: with pkgs; with stdenv; @@ -7,18 +7,17 @@ let gradle = gradle_4_10; targetAndroid = { "android" = true; - "" = true; + "all" = true; }.${target-os} or false; targetIOS = { "ios" = true; - "" = true; + "all" = true; }.${target-os} or false; in { buildInputs = - [ bundler ruby ] ++ ## bundler/ruby used for fastlane - lib.optional targetAndroid [ + lib.optionals targetAndroid [ openjdk gradle ]; shellHook = diff --git a/nix/shell.sh b/nix/shell.sh index 497ea385fb..7fe3fd47bc 100755 --- a/nix/shell.sh +++ b/nix/shell.sh @@ -22,8 +22,8 @@ fi if command -v "nix" >/dev/null 2>&1; then echo -e "${GREEN}Configuring Nix shell...${NC}"; if [[ $@ == "ENTER_NIX_SHELL" ]]; then - exec nix-shell + exec nix-shell --show-trace else - exec nix-shell --run "$@" + exec nix-shell --show-trace --run "$@" fi fi diff --git a/nix/status-go/gomobile/default.nix b/nix/status-go/gomobile/default.nix index 9464cbb5ec..b9415f8f0a 100644 --- a/nix/status-go/gomobile/default.nix +++ b/nix/status-go/gomobile/default.nix @@ -26,7 +26,7 @@ in buildGoPackage rec { ''; patches = [ ./ndk-search-path.patch ./resolve-nix-android-sdk.patch ] - ++ lib.optional isDarwin [ ./ignore-nullability-error-on-ios.patch ]; + ++ lib.optional isDarwin ./ignore-nullability-error-on-ios.patch; postPatch = '' substituteInPlace cmd/gomobile/install.go --replace "\`adb\`" "\`${platform-tools}/bin/adb\`" @@ -41,14 +41,14 @@ in buildGoPackage rec { mkdir -p $out $bin/lib ln -s ${ncurses5}/lib/libncursesw.so.5 $bin/lib/libtinfo.so.5 - '' + lib.optionalString isDarwin '' + '' + (if isDarwin then '' wrapProgram $bin/bin/gomobile \ --prefix "PATH" : "${lib.makeBinPath [ xcodeWrapper ]}" \ --prefix "LD_LIBRARY_PATH" : "${lib.makeLibraryPath [ ncurses5 zlib ]}:$bin/lib" - '' + lib.optionalString (!isDarwin) '' + '' else '' wrapProgram $bin/bin/gomobile \ --prefix "LD_LIBRARY_PATH" : "${lib.makeLibraryPath [ ncurses5 zlib ]}:$bin/lib" - '' + '' + '') + '' $bin/bin/gomobile init ''; diff --git a/scripts/run-environment-check.sh b/scripts/run-environment-check.sh index 179572bf74..1e002c39a6 100755 --- a/scripts/run-environment-check.sh +++ b/scripts/run-environment-check.sh @@ -40,7 +40,7 @@ fi if [ "$PLATFORM" == 'android' ]; then if [ ! -d $ANDROID_SDK_ROOT ]; then - echo -e "${GREEN}NDK setup not complete, please run 'scripts/setup'!${NC}" + echo -e "${GREEN}SDK setup not complete, please run 'scripts/setup'!${NC}" exit 1 fi if [ ! -d $ANDROID_NDK_ROOT ]; then diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000..c62bba5c40 --- /dev/null +++ b/shell.nix @@ -0,0 +1,41 @@ +{ pkgs ? import {}, + target-os ? "all" }: +with pkgs; + +let + projectDeps = import ./default.nix { inherit target-os; }; + targetMobile = { + "android" = true; + "ios" = true; + "all" = true; + }.${target-os} or false; + # TODO: Try to use stdenv for iOS. The problem is with building iOS as the build is trying to pass parameters to Apple's ld that are meant for GNU's ld (e.g. -dynamiclib) + _stdenv = stdenvNoCC; + _mkShell = mkShell.override { stdenv = _stdenv; }; + +in _mkShell { + buildInputs = [ + # utilities + bash + curl + git + jq + ncurses + lsof # used in scripts/start-react-native.sh + ps # used in scripts/start-react-native.sh + unzip + wget + ] ++ lib.optionals targetMobile [ bundler ruby ]; # bundler/ruby used for fastlane + inputsFrom = [ projectDeps ]; + shellHook = + '' + set -e + '' + + projectDeps.shellHook + + '' + if [ -n "$ANDROID_SDK_ROOT" ] && [ ! -d "$ANDROID_SDK_ROOT" ]; then + ./scripts/setup # we assume that if the Android SDK dir does not exist, setup script needs to be run + fi + set +e + ''; +} \ No newline at end of file