Fix issues found when running `make setup` on clean MacOS and Linux machines

ci/common.groovy

@@ -93,7 +93,6 @@ def installJSDeps(platform) {
   def maxAttempts = 10
   def installed = false
   /* prepare environment for specific platform build */
-  sh "scripts/run-environment-check.sh ${platform}"
   sh "scripts/prepare-for-platform.sh ${platform}"
   while (!installed && attempt <= maxAttempts) {
     println "#${attempt} attempt to install npm deps"

docker/android/Makefile

@@ -24,13 +24,12 @@ IMAGE_NAME = statusteam/status-build-android:$(IMAGE_TAG)
 
 build: $(ANDROID_NDK_ARCHIVE) $(ANDROID_SDK_ARCHIVE)
 	docker build \
-        --build-arg="BASE_IMAGE_TAG=$(BASE_IMAGE_TAG)" \
+		--build-arg="ANDROID_NDK_VERSION=$(ANDROID_NDK_VERSION)" \
-        --build-arg="ANDROID_NDK_VERSION=$(ANDROID_NDK_VERSION)" \
+		--build-arg="ANDROID_SDK_VERSION=$(ANDROID_SDK_VERSION)" \
-        --build-arg="ANDROID_SDK_VERSION=$(ANDROID_SDK_VERSION)" \
+		--build-arg="SDK_PLATFORM_VERSION=$(SDK_PLATFORM_VERSION)" \
-        --build-arg="SDK_PLATFORM_VERSION=$(SDK_PLATFORM_VERSION)" \
+		--build-arg="SDK_BUILD_TOOLS_VERSION=$(SDK_BUILD_TOOLS_VERSION)" \
-        --build-arg="SDK_BUILD_TOOLS_VERSION=$(SDK_BUILD_TOOLS_VERSION)" \
+		--label="commit=$(GIT_COMMIT)" \
-        --label="commit=$(GIT_COMMIT)" \
+		-t $(IMAGE_NAME) .
-        -t $(IMAGE_NAME) .
 
 $(ANDROID_NDK_ARCHIVE):
 	wget -q "$(ANDROID_NDK_URL)" -O "$(ANDROID_NDK_ARCHIVE)"
@@ -42,11 +41,11 @@ $(ANDROID_SDK_ARCHIVE):
 
 test: ## Run build inside the image as a test
 	docker run -u $(shell id -u):$(shell id -g) \
-        --name android-test --rm \
+		--name android-test --rm \
-        --tmpfs /var/tmp:rw,size=1G,exec,mode=1777 \
+		--tmpfs /var/tmp:rw,size=1G,exec,mode=1777 \
-        -v $(GIT_ROOT):/repo:rw \
+		-v $(GIT_ROOT):/repo:rw \
-        -w /repo $(IMAGE_NAME) \
+		-w /repo $(IMAGE_NAME) \
-        docker/android/build.sh
+		docker/android/build.sh
 
 push: build
 	docker push $(IMAGE_NAME)

scripts/generate-keystore.sh

@@ -17,5 +17,7 @@ STATUS_RELEASE_KEY_PASSWORD=$(property_gradle 'STATUS_RELEASE_KEY_PASSWORD')
 [[ -e "${STORE_FILE/#\~/$HOME}" ]] && echo "Keystore $STORE_FILE already exists, please manually remove it if you want to regenerate." && exit 0
 
 echo "Generating keystore $STORE_FILE"
+keydirname="$( dirname "$STORE_FILE" )"
+[ -d $keydirname ] || mkdir -p $keydirname
 keytool -genkey -v -keystore ${STORE_FILE}  -keyalg RSA -keysize 2048 -validity 10000 -alias ${STATUS_RELEASE_KEY_ALIAS} \
         -storepass ${STATUS_RELEASE_STORE_PASSWORD} -keypass ${STATUS_RELEASE_KEY_PASSWORD} -dname "CN=, OU=, O=, L=, S=, C="

scripts/lib/setup/installers.sh

@@ -40,10 +40,14 @@ function install_and_setup_package_manager() {
       cmake
       curl
       g++
+      lib32ncurses5    # required for Android SDK
+      lib32stdc++6     # required for Android SDK
       libssl-dev
       libtool
       make
+      pkg-config       # required to e.g. build watchman
       python-dev
+      rlwrap           # required to use clj
       wget
       unzip
     )
@@ -150,6 +154,7 @@ function install_watchman() {
       local current_dir=$(pwd)
       local clone_path="/tmp/watchman"
 
+      rm -rf $clone_path
       git clone https://github.com/facebook/watchman.git $clone_path
       cd $clone_path
       git checkout v$required_version
@@ -180,9 +185,31 @@ function install_homebrew_if_needed() {
   fi
 }
 
+function export_android_sdk_vars() {
+  local profile
+  local target_path
+  if is_macos; then
+    profile=$HOME/.bash_profile
+  elif is_linux; then
+    profile=$HOME/.bashrc
+  fi
+
+  [ -f $profile ] || touch $profile
+  if ! grep -Fq "export ANDROID_SDK_ROOT=" $profile; then
+    echo "export ANDROID_HOME=\"$1\"" >> $profile && \
+    echo "export ANDROID_SDK_ROOT=\"$1\"" >> $profile && \
+    echo "export PATH=\"$1/tools:$1/tools/bin:\$PATH\"" >> $profile
+  fi
+  export ANDROID_HOME="$1" && \
+  export ANDROID_SDK_ROOT="$1" && \
+  export PATH="$1/tools:$1/tools/bin:$PATH"
+}
+
 function install_android_sdk() {
   if is_macos; then
     brew_cask_install android-sdk
+
+    [ -z "$ANDROID_SDK_ROOT" ] && export_android_sdk_vars /usr/local/share/android-sdk
   elif is_linux; then
     install_android_sdk_linux
   fi
@@ -191,7 +218,33 @@ function install_android_sdk() {
 }
 
 function install_android_sdk_linux() {
-  cecho "@b@yellow[[+ Skipping Android SDK setup, not implemented on Linux]]"
+  if [ -z "$ANDROID_SDK_ROOT" ]; then
+    if grep -Fq "sdk.dir" $_localPropertiesPath; then
+      local _sdkParentDir="$(awk -F'=' "/^sdk.dir=/{print \$2}" "$_localPropertiesPath")"
+      export_android_sdk_vars $_sdkParentDir
+      cecho "@green[[Android SDK already declared.]]"
+    else
+      local required_version=$(toolversion android-sdk)
+      local _sdkParentDir=$HOME/Android/Sdk
+      mkdir -p $_sdkParentDir
+      cecho "@cyan[[Downloading Android SDK.]]"
+
+      downloadUrl . sdk-tools-linux.zip https://dl.google.com/android/repository/sdk-tools-linux-${required_version}.zip && \
+        cecho "@cyan[[Extracting Android SDK to $_sdkParentDir.]]" && \
+        unzip -q -o ./sdk-tools-linux.zip -d "$_sdkParentDir" && \
+        rm -f ./sdk-tools-linux.zip && \
+        _sdkTargetDir="$_sdkParentDir" && \
+        echo "sdk.dir=$_sdkTargetDir" | tee -a $_localPropertiesPath && \
+        export_android_sdk_vars $_sdkParentDir && \
+        cecho "@blue[[Android SDK installation completed in $_sdkTargetDir.]]" || \
+        return 0
+    fi
+  else
+    if ! grep -Fq "sdk.dir" $_localPropertiesPath; then
+      echo "sdk.dir=$ANDROID_SDK_ROOT" | tee -a $_localPropertiesPath
+    fi
+    cecho "@green[[Android SDK already declared.]]"
+  fi
 
   return 1
 }
@@ -243,11 +296,13 @@ function install_nvm() {
   if ! program_version_exists 'nvm' "$required_version"; then
     cecho "@b@blue[[+ Installing nvm $required_version]]"
 
+    # Ensure a profile file exists, otherwise NVM will not add its configuration anywhere
+    # and will therefore be inaccessible
+    [ -f "~/.bash_profile" ] || touch ~/.bash_profile
+
     sudo apt install -y build-essential libssl-dev
     source scripts/3rd-party/nvm/${required_version}/install.sh
-    export NVM_DIR="$HOME/.nvm"
+    load_nvm_if_available
-    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
-    [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
   else
     cecho "+ nvm already installed... skipping."
   fi
@@ -357,7 +412,7 @@ function dependency_setup() {
   echo
 
   cd "$(repo_path)"
-  eval "$@" || (cecho "@b@red[[Error running dependency install]]" && exit 1)
+  eval "$@" || (cecho "@b@red[[Error running dependency install '$@']]" && exit 1)
 
   echo
   echo "  + done"
@@ -370,10 +425,11 @@ function use_android_sdk() {
       echo "sdk.dir=$ANDROID_SDK_ROOT" | tee -a $_localPropertiesPath
     fi
 
-    local ANDROID_BUILD_TOOLS_VERSION=$(toolversion ANDROID_BUILD_TOOLS_VERSION)
+    local ANDROID_BUILD_TOOLS_VERSION=$(toolversion android-sdk-build-tools)
-    local ANDROID_PLATFORM_VERSION=$(toolversion ANDROID_PLATFORM_VERSION)
+    local ANDROID_PLATFORM_VERSION=$(toolversion android-sdk-platform)
     touch ~/.android/repositories.cfg
     echo y | sdkmanager "platform-tools" "build-tools;$ANDROID_BUILD_TOOLS_VERSION" "platforms;$ANDROID_PLATFORM_VERSION"
+    yes | sdkmanager --licenses
   else
     local _docUrl="https://status.im/build_status/"
     cecho "@yellow[[ANDROID_SDK_ROOT environment variable not defined, please install the Android SDK.]]"
@@ -383,22 +439,24 @@ function use_android_sdk() {
 
     exit 1
   fi
+
+  scripts/generate-keystore.sh
 }
 
 function install_android_ndk() {
   if grep -Fq "ndk.dir" $_localPropertiesPath; then
     cecho "@green[[Android NDK already declared.]]"
   else
+    local ANDROID_NDK_VERSION=$(toolversion android-ndk)
     local _ndkParentDir=~/Android/Sdk
     mkdir -p $_ndkParentDir
     cecho "@cyan[[Downloading Android NDK.]]"
 
-    PLATFORM="linux"
+    local PLATFORM="linux"
-    if [ "$(uname)" == "Darwin" ]; then # we run osx
+    if is_macos; then
         PLATFORM="darwin"
     fi
 
-    local ANDROID_NDK_VERSION=$(toolversion ANDROID_NDK_VERSION)
     downloadUrl . android-ndk.zip https://dl.google.com/android/repository/android-ndk-$ANDROID_NDK_VERSION-$PLATFORM-x86_64.zip && \
       cecho "@cyan[[Extracting Android NDK to $_ndkParentDir.]]" && \
       unzip -q -o ./android-ndk.zip -d "$_ndkParentDir" && \

scripts/lib/setup/packages.sh

@@ -202,9 +202,10 @@ function load_rvm_if_available() {
 ###############
 
 function load_nvm_if_available() {
-  [ -f ~/.nvm/nvm.sh ] && source ~/.nvm/nvm.sh
+  local nvm_path=${NVM_DIR:-~/.nvm}
+  [ -f ${nvm_path}/nvm.sh ] && chmod +x ${nvm_path}/nvm.sh && source ${nvm_path}/nvm.sh
 }
 
 function nvm_installed() {
-  program_exists "nvm"
+  declare -F nvm &>/dev/null
 }

scripts/prepare-for-platform.sh

@@ -31,7 +31,7 @@ fi
 
 scripts/run-environment-check.sh $1
 
-echo "Creating link: package.json -> ${PLATFORM_FOLDER}/package.json.orig "
+echo "Creating link: package.json -> ${PLATFORM_FOLDER}/package.json.orig"
 ln -sf ${PLATFORM_FOLDER}/package.json.orig package.json
 
 echo "Creating link: yarn.lock -> ${PLATFORM_FOLDER}/yarn.lock"

scripts/run-environment-check.sh

@@ -14,8 +14,7 @@ source_lib "packages.sh"
 EXPECTED_NODE_VERSION="v$(toolversion node)" # note the 'v' in front, that is how node does versioning
 EXPECTED_YARN_VERSION="$(toolversion yarn)" # note the lack of 'v' in front. inconsistent. :(
 
-#if no arguments passed, inform user about possible ones
+# if no arguments passed, inform user about possible ones
-
 if [ $# -eq 0 ]; then
   echo -e "${GREEN}This script should be invoked with platform argument: 'android', 'ios' or 'desktop'${NC}"
   exit 1
@@ -23,12 +22,13 @@ else
   PLATFORM=$1
 fi
 
-if ! program_version_exists node $EXPECTED_NODE_VERSION || ! program_exists yarn $EXPECTED_YARN_VERSION; then
+load_nvm_if_available
-  echo -e "${YELLOW}**********************************************************************************************"
+
+if ! program_exists node || ! program_exists yarn; then
+  echo -e "${YELLOW}********************************************************************************************"
 
   nvmrc="./.nvmrc"
-  if [ -e "$nvmrc" ]; then
+  if [ -e "$nvmrc" ] && nvm_installed; then
-    node_version=$(node -v)
     version_alias=$(cat "$nvmrc")
     echo -e "Please run 'nvm use $version_alias' in the terminal and try again."
   else
@@ -56,7 +56,7 @@ fi
 
 if [[ $PLATFORM == 'android' ]]; then
   _localPropertiesPath=./android/local.properties
-  if ! grep -Fq "ndk.dir" $_localPropertiesPath > /dev/null; then
+  if [ ! -f $_localPropertiesPath ] || ! grep -Fq "ndk.dir" $_localPropertiesPath > /dev/null; then
     if [ -z $ANDROID_NDK_HOME ]; then
       echo -e "${GREEN}NDK directory not configured, please run 'make setup' or add the line to ${_localPropertiesPath}!${NC}"
       exit 1
@@ -64,4 +64,8 @@ if [[ $PLATFORM == 'android' ]]; then
   fi
 fi
 
-echo -e "${GREEN}Finished!${NC}"
+if [[ $PLATFORM == 'setup' ]]; then
+  echo -e "${YELLOW}Finished! Please close your terminal, and reopen a new one before building Status.${NC}"
+else
+  echo -e "${GREEN}Finished!${NC}"
+fi

scripts/toolversion

@@ -33,9 +33,9 @@ if [[ -z "${1}" ]]; then usage; fi
 NAME=${1}
 
 getColumn () {
-    local out=$(awk -F';' "/^${NAME};/{print \$${1}}" "${TOOL_VERSIONS_FILE}")
+    local value=$(awk -F';' "/^${NAME};/{print \$${1}}" "${TOOL_VERSIONS_FILE}")
-    [ -z "$out" ] && exit 1
+    [ -z "$value" ] && echo "\nUnexpected missing value for ${NAME} in ${TOOL_VERSIONS_FILE}" && exit 1
-    echo "$out"
+    echo $value
 }
 
 if [[ $CHECKSUM ]]; then