change .TOOLVERSION into csv, add toolversion script

Signed-off-by: Jakub Sokołowski <jakub@status.im>
Signed-off-by: Pedro Pombeiro <pombeirp@users.noreply.github.com>

.TOOLVERSIONS

@@ -1,12 +1,12 @@
-ANDROID_BUILD_TOOLS_VERSION=28.0.1
+ANDROID_BUILD_TOOLS_VERSION;28.0.1;
-ANDROID_NDK_VERSION=r10e
+ANDROID_NDK_VERSION;r10e;
-ANDROID_PLATFORM_VERSION=android-27
+ANDROID_PLATFORM_VERSION;android-27;
-clojure_cli=1.9.0.381
+clojure_cli;1.9.0.381;
-conan=1.9.0
+conan;1.9.0;
-leiningen=2.8.1
+leiningen;2.8.1;
-maven=3.5.2
+maven;3.5.2;
-node=10.12.0
+node;10.12.0;
-nvm=0.33.11
+nvm;0.33.11;
-react_native_cli=2.0.1
+react_native_cli;2.0.1;
-watchman=4.9.0
+watchman;4.9.0;
-yarn=1.13.0
+yarn;1.13.0;

Makefile

@@ -4,7 +4,7 @@ help: ##@other Show this help
 	@perl -e '$(HELP_FUN)' $(MAKEFILE_LIST)
 
 DO_SPACE_URL = https://status-go.ams3.digitaloceanspaces.com
-GITHUB_URL = https://github.com/status-im/status-go/releases/download
+GITHUB_URL = https://github.com/status-im/status-go/releases
 RCTSTATUS_DIR = modules/react-native-status/ios/RCTStatus
 ANDROID_LIBS_DIR = android/app/libs
 STATUS_GO_VER = $(shell cat STATUS_GO_VERSION)
@@ -35,7 +35,7 @@ HELP_FUN = \
 			   print "\n"; \
 		   }
 
-__read__toolversion__ = $(shell grep $(1) ./.TOOLVERSIONS | cut -d'=' -f2-)
+__toolversion = $(shell $(GIT_ROOT)/scripts/toolversion $(1))
 
 # Main targets
 
@@ -57,10 +57,15 @@ $(STATUS_GO_IOS_ARCH):
 	if [ $$? -ne 0 ]; then \
 		echo "Failed to download from DigitalOcean Bucket, checking GitHub..."; \
 		curl --fail --silent --location \
-			"$(GITHUB_URL)/$(STATUS_GO_VER)/status-go-ios.zip" \
+			"$(GITHUB_URL)/download/$(STATUS_GO_VER)/status-go-ios.zip" \
 			--output "$(STATUS_GO_IOS_ARCH)"; \
 		if [ $$? -ne 0 ]; then \
 			echo "Failed to download from GitHub!"; \
+			echo "Please check the contents of your STATUS_GO_VERSION are correct."; \
+			echo "Verify the version has been uploaded:"; \
+			echo " * $(DO_SPACE_URL)/index.html"; \
+			echo " * $(GITHUB_URL)"; \
+			exit 1; \
 		fi \
 	fi
 

ci/common.groovy

@@ -7,7 +7,7 @@ def version() {
 def getToolVersion(name) {
   def version = sh(
     returnStdout: true,
-    script: "grep ${name} ${env.WORKSPACE}/.TOOLVERSIONS | cut -d'=' -f2-"
+    script: "${env.WORKSPACE}/scripts/toolversion ${name}"
   ).trim()
   return version
 }

docker/linux/Makefile

@@ -1,3 +1,5 @@
+__toolversion = $(shell $(GIT_ROOT)/scripts/toolversion $(1))
+
 GIT_COMMIT = $(shell git rev-parse --short HEAD)
 
 # WARNING: Change version in Dockerfile too
@@ -8,7 +10,6 @@ QT_MD5SUM  = 974fda61267cfb6e45984ee5f0a285f8
 QT_URL = https://download.qt.io/archive/qt
 QT_CI_COMMIT_SHA = 55ffd9f225708b3aa1443851cfa8dead2c1f9959
 REACT_NATIVE_DESKTOP_COMMIT_SHA = 2a76c435a27f0bf6c089f74222212b3a58e10385
-__read__toolversion__ = $(shell grep $(1) ../../../.TOOLVERSIONS | cut -d'=' -f2-)
 
 # WARNING: Remember to change the tag when updating the image
 IMAGE_TAG = 1.1.1
@@ -22,10 +23,10 @@ build: $(QT_ARCHIVE)
 		--build-arg="QT_VERSION=$(QT_VERSION)" \
 		--build-arg="QT_CI_COMMIT_SHA=$(QT_CI_COMMIT_SHA)" \
 		--build-arg="REACT_NATIVE_DESKTOP_COMMIT_SHA=$(REACT_NATIVE_DESKTOP_COMMIT_SHA)" \
-		--build-arg="LEIN_VERSION=$(call __read__toolversion__,lein)" \
+		--build-arg="LEIN_VERSION=$(call __toolversion,lein)" \
-		--build-arg="NODE_VERSION=$(call __read__toolversion__,node)" \
+		--build-arg="NODE_VERSION=$(call __toolversion,node)" \
-		--build-arg="NVM_VERSION=$(call __read__toolversion__,nvm)" \
+		--build-arg="NVM_VERSION=$(call __toolversion,nvm)" \
-		--build-arg="YARN_VERSION=$(call __read__toolversion__,yarn)" \
+		--build-arg="YARN_VERSION=$(call __toolversion,yarn)" \
 		--label="commit=$(GIT_COMMIT)" \
 		-t $(IMAGE_NAME) .; \
 	rm -rf ./scripts

docker/windows/Makefile

@@ -1,3 +1,5 @@
+__toolversion = $(shell $(GIT_ROOT)/scripts/toolversion $(1))
+
 GIT_COMMIT = $(shell git rev-parse --short HEAD)
 
 # WARNING: Remember to change the tag when updating the image
@@ -5,7 +7,6 @@ IMAGE_TAG = 1.1.1
 IMAGE_NAME = statusteam/windows-desktop-ubuntu:$(IMAGE_TAG)
 
 REACT_NATIVE_DESKTOP_COMMIT_SHA = 2a76c435a27f0bf6c089f74222212b3a58e10385
-__read__toolversion__ = $(shell grep $(1) ../../../.TOOLVERSIONS | cut -d'=' -f2-)
 
 build:
 	rm -rf ./scripts && \
@@ -13,10 +14,10 @@ build:
 	cp -R ../../scripts/3rd-party/ ./scripts/ && \
 	docker build \
 		--build-arg="REACT_NATIVE_DESKTOP_COMMIT_SHA=$(REACT_NATIVE_DESKTOP_COMMIT_SHA)" \
-		--build-arg="CONAN_VERSION=$(call __read__toolversion__,conan)" \
+		--build-arg="CONAN_VERSION=$(call __toolversion,conan)" \
-		--build-arg="LEIN_VERSION=$(call __read__toolversion__,lein)" \
+		--build-arg="LEIN_VERSION=$(call __toolversion,lein)" \
-		--build-arg="NODE_VERSION=$(call __read__toolversion__,node)" \
+		--build-arg="NODE_VERSION=$(call __toolversion,node)" \
-		--build-arg="NVM_VERSION=$(call __read__toolversion__,nvm)" \
+		--build-arg="NVM_VERSION=$(call __toolversion,nvm)" \
 		--label="commit=$(GIT_COMMIT)" \
 		-t $(IMAGE_NAME) .; \
 	rm -rf ./scripts

scripts/build-desktop.sh

@@ -126,7 +126,7 @@ function init() {
         fi
 
         echo "${RED}Conan package manager not found. Installing...${NC}"
-        pip3 install conan==$(get_tool_version conan)
+        pip3 install conan==$(toolversion conan)
       fi
 
       conan remote add --insert 0 -f status-im https://conan.status.im

scripts/lib/setup/installers.sh

@@ -84,7 +84,7 @@ EOF
 
 function install_leiningen() {
   if is_macos; then
-    brew_install leiningen $(get_tool_version leiningen) f7e10afc6d04a13d28e825db71326d16c12e9724
+    brew_install leiningen $(toolversion leiningen) f7e10afc6d04a13d28e825db71326d16c12e9724
   elif is_linux; then
     install_leiningen_linux
   fi
@@ -92,7 +92,7 @@ function install_leiningen() {
 
 function install_leiningen_linux() {
   local destination=/usr/bin/lein
-  local required_version=$(get_tool_version leiningen)
+  local required_version=$(toolversion leiningen)
   if ! program_version_exists "lein" "$required_version"; then
     cecho "@b@blue[[+ Installing lein...]]"
 
@@ -112,7 +112,7 @@ EOF
 
 function install_clojure_cli() {
   if is_macos; then
-    local required_version=$(get_tool_version clojure_cli)
+    local required_version=$(toolversion clojure_cli)
     brew_install clojure $required_version 90ea0cb4b194282b5906108dcec522c5a1ed7ce0
   elif is_linux; then
     install_clojure_cli_linux
@@ -120,7 +120,7 @@ function install_clojure_cli() {
 }
 
 function install_clojure_cli_linux() {
-  local required_version=$(get_tool_version clojure_cli)
+  local required_version=$(toolversion clojure_cli)
 
   if ! program_exists "clojure" || ! echo "$(clj -r <<< '(System/exit 0)')" | grep -q -o ${required_version%.*}; then
     cecho "@b@blue[[+ Installing Clojure CLI...]]"
@@ -142,7 +142,7 @@ EOF
 }
 
 function install_watchman() {
-  local required_version=$(get_tool_version watchman)
+  local required_version=$(toolversion watchman)
   if is_macos; then
     brew_install watchman $required_version 1a41406af8db6bbc0c94811cf60043a7436be3c4
   elif is_linux; then
@@ -197,7 +197,7 @@ function install_android_sdk_linux() {
 }
 
 function install_maven() {
-  local required_version=$(get_tool_version maven)
+  local required_version=$(toolversion maven)
   brew_install maven $required_version 4c23c22dc71eadaeb7b25d6e6c10fd53bfc26976
   linux_install maven
 }
@@ -206,7 +206,7 @@ function install_react_native_cli() {
   cd "$(repo_path)"
 
   local npm_command='npm'
-  local required_version=$(get_tool_version react_native_cli)
+  local required_version=$(toolversion react_native_cli)
 
   if is_linux && ! nvm_installed; then
     # aptitude version of node requires sudo for global install
@@ -221,7 +221,7 @@ function install_react_native_cli() {
 }
 
 function install_yarn() {
-  local required_version=$(get_tool_version yarn)
+  local required_version=$(toolversion yarn)
   if ! program_exists "yarn"; then
     npm install -g yarn@$required_version # Install the required yarn version
   else
@@ -238,7 +238,7 @@ function install_yarn() {
 }
 
 function install_nvm() {
-  local required_version=$(get_tool_version nvm)
+  local required_version=$(toolversion nvm)
 
   if ! program_version_exists 'nvm' "$required_version"; then
     cecho "@b@blue[[+ Installing nvm $required_version]]"
@@ -255,7 +255,7 @@ function install_nvm() {
 
 function install_node_via_nvm() {
   local nvmrc="$(repo_path)/.nvmrc"
-  local required_version=$(get_tool_version node)
+  local required_version=$(toolversion node)
 
   cd "$(repo_path)"
 
@@ -281,7 +281,7 @@ function install_node_via_nvm() {
 }
 
 function install_node_via_package_manager() {
-  local required_version=$(get_tool_version node)
+  local required_version=$(toolversion node)
   if ! program_version_exists "node" "v$required_version"; then
     if is_macos; then
       brew_install node "$required_version" b801cc6b71e7c09448b4f823e493710665de68eb
@@ -370,8 +370,8 @@ function use_android_sdk() {
       echo "sdk.dir=$ANDROID_SDK_ROOT" | tee -a $_localPropertiesPath
     fi
 
-    local ANDROID_BUILD_TOOLS_VERSION=$(get_tool_version ANDROID_BUILD_TOOLS_VERSION)
+    local ANDROID_BUILD_TOOLS_VERSION=$(toolversion ANDROID_BUILD_TOOLS_VERSION)
-    local ANDROID_PLATFORM_VERSION=$(get_tool_version ANDROID_PLATFORM_VERSION)
+    local ANDROID_PLATFORM_VERSION=$(toolversion ANDROID_PLATFORM_VERSION)
     touch ~/.android/repositories.cfg
     echo y | sdkmanager "platform-tools" "build-tools;$ANDROID_BUILD_TOOLS_VERSION" "platforms;$ANDROID_PLATFORM_VERSION"
   else
@@ -398,7 +398,7 @@ function install_android_ndk() {
         PLATFORM="darwin"
     fi
 
-    local ANDROID_NDK_VERSION=$(get_tool_version ANDROID_NDK_VERSION)
+    local ANDROID_NDK_VERSION=$(toolversion ANDROID_NDK_VERSION)
     downloadUrl . android-ndk.zip https://dl.google.com/android/repository/android-ndk-$ANDROID_NDK_VERSION-$PLATFORM-x86_64.zip && \
       cecho "@cyan[[Extracting Android NDK to $_ndkParentDir.]]" && \
       unzip -q -o ./android-ndk.zip -d "$_ndkParentDir" && \

scripts/lib/setup/packages.sh

@@ -24,18 +24,10 @@ function program_version_exists() {
   $(exit 1)
 }
 
-function get_tool_version() {
+function toolversion() {
-  local toolName=$1
+  ./toolversion "${1}"
-  local version=`grep "^$toolName" ./.TOOLVERSIONS | cut -d'=' -f2-`
-  if [ -z "$version" ]; then
-    exit 1
-  fi
-
-  echo $version
 }
 
-
-
 ########
 # Homebrew
 ########

scripts/run-environment-check.sh

@@ -11,8 +11,8 @@ source "$_current_dir/lib/setup/path-support.sh"
 
 source_lib "packages.sh"
 
-EXPECTED_NODE_VERSION="v$(get_tool_version node)" # note the 'v' in front, that is how node does versioning
+EXPECTED_NODE_VERSION="v$(toolversion node)" # note the 'v' in front, that is how node does versioning
-EXPECTED_YARN_VERSION="$(get_tool_version yarn)" # note the lack of 'v' in front. inconsistent. :(
+EXPECTED_YARN_VERSION="$(toolversion yarn)" # note the lack of 'v' in front. inconsistent. :(
 
 #if no arguments passed, inform user about possible ones
 

scripts/toolversion

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+################################################################################
+# This tool fetches versions and checksums of build tools from the .TOOLVERSIONS
+# file in project root. This is then used by various setup scripts,
+# and most importantly by Dockerfiles.
+################################################################################
+
+set -e
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+TOOL_VERSIONS_FILE="${GIT_ROOT}/.TOOLVERSIONS"
+
+usage () {
+  echo "Usage: toolversion [-c] <name>" >&2
+  echo
+  echo "This script extract tooling versions from ${TOOL_VERSIONS_FILE}"
+  exit 0
+}
+
+# some options parsing
+while getopts ":ch" opt; do
+  case $opt in
+    c)  CHECKSUM=1; shift ;;
+    h)  usage;;
+    \?) echo "Invalid option: -$OPTARG" >&2; exit 1;;
+  esac
+done
+
+# verify the main argument was given
+if [[ -z "${1}" ]]; then usage; fi
+
+NAME=${1}
+
+getColumn () {
+    awk -F';' "/^${NAME};/{print \$${1}}" "${TOOL_VERSIONS_FILE}"
+}
+
+if [[ $CHECKSUM ]]; then
+    getColumn 3
+else
+    getColumn 2
+fi