From c5f74ac2c03f35e076e8a40edadc4e6874114e19 Mon Sep 17 00:00:00 2001 From: "Michael Bradley, Jr" Date: Thu, 15 Apr 2021 16:14:30 -0500 Subject: [PATCH] build: use GitHub credentials when downloading bottles for macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub SokoĊ‚owski --- Makefile | 39 ++++--------------- ci/Jenkinsfile.macos | 10 ++++- scripts/fetch-brew-bottle.sh | 73 ++++++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 32 deletions(-) create mode 100755 scripts/fetch-brew-bottle.sh diff --git a/Makefile b/Makefile index e51ddd771e..d5dd84da9e 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,6 @@ BUILD_SYSTEM_DIR := vendor/nimbus-build-system .PHONY: \ all \ bottles \ - bottles-dummy \ - bottles-macos \ check-pkg-target-linux \ check-pkg-target-macos \ check-pkg-target-windows \ @@ -66,7 +64,6 @@ else endif ifeq ($(detected_OS),Darwin) - BOTTLES_TARGET := bottles-macos CFLAGS := -mmacosx-version-min=10.14 export CFLAGS CGO_CFLAGS := -mmacosx-version-min=10.14 @@ -77,7 +74,6 @@ ifeq ($(detected_OS),Darwin) PKG_TARGET := pkg-macos RUN_TARGET := run-macos else ifeq ($(detected_OS),Windows) - BOTTLES_TARGET := bottles-dummy LIBSTATUS_EXT := dll PKG_TARGET := pkg-windows QRCODEGEN_MAKE_PARAMS := CC=gcc @@ -86,7 +82,6 @@ else ifeq ($(detected_OS),Windows) VCINSTALLDIR ?= C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\ export VCINSTALLDIR else - BOTTLES_TARGET := bottles-dummy LIBSTATUS_EXT := so PKG_TARGET := pkg-linux RUN_TARGET := run-linux @@ -107,33 +102,15 @@ ifneq ($(detected_OS),Windows) $(error The pkg-windows target must be run on Windows) endif -bottles: $(BOTTLES_TARGET) +ifeq ($(detected_OS),Darwin) +bottles/openssl: + ./scripts/fetch-brew-bottle.sh openssl -bottles-dummy: ; +bottles/pcre: + ./scripts/fetch-brew-bottle.sh pcre -BOTTLE_OPENSSL := bottles/openssl/INSTALL_RECEIPT.json - -$(BOTTLE_OPENSSL): - echo -e "\e[92mFetching:\e[39m bottles for macOS" - rm -rf bottles/Downloads/openssl* bottles/openssl* - mkdir -p bottles/Downloads - cd bottles/Downloads && \ - curl -L -o openssl.tar.gz -u _:_ $$(brew info --json=v1 openssl | jq -r '.[0].bottle.stable.files.mojave.url') && \ - tar xzf openssl.tar.gz && \ - mv openssl*/* ../openssl - -BOTTLE_PCRE := bottles/pcre/INSTALL_RECEIPT.json - -$(BOTTLE_PCRE): - rm -rf bottles/Downloads/pcre* bottles/pcre* - mkdir -p bottles/Downloads - cd bottles/Downloads && \ - curl -L -o pcre.tar.gz -u _:_ $$(brew info --json=v1 pcre | jq -r '.[0].bottle.stable.files.mojave.url') && \ - tar xzf pcre.tar.gz && \ - mv pcre*/* ../pcre - -bottles-macos: | $(BOTTLE_OPENSSL) $(BOTTLE_PCRE) - rm -rf bottles/Downloads +bottles: bottles/openssl bottles/pcre +endif deps: | deps-common bottles @@ -438,7 +415,7 @@ pkg-macos: check-pkg-target-macos $(STATUS_CLIENT_DMG) pkg-windows: check-pkg-target-windows $(STATUS_CLIENT_ZIP) clean: | clean-common - rm -rf bin/* node_modules pkg/* tmp/* $(STATUSGO) + rm -rf bin/* node_modules bottles/* pkg/* tmp/* $(STATUSGO) + $(MAKE) -C vendor/DOtherSide/build --no-print-directory clean run: rcc $(RUN_TARGET) diff --git a/ci/Jenkinsfile.macos b/ci/Jenkinsfile.macos index bb348fd58e..4ef7e78ebf 100644 --- a/ci/Jenkinsfile.macos +++ b/ci/Jenkinsfile.macos @@ -41,7 +41,15 @@ pipeline { includes: '**/*', path: 'vendor/nimbus-build-system/vendor/Nim/bin' ]]) { - sh 'make deps' + withCredentials([ + usernamePassword( /* For fetching HomeBrew bottles. */ + credentialsId: "status-im-auto-pkgs", + usernameVariable: 'GITHUB_USER', + passwordVariable: 'GITHUB_TOKEN' + ) + ]) { + sh 'make deps' + } } } } diff --git a/scripts/fetch-brew-bottle.sh b/scripts/fetch-brew-bottle.sh new file mode 100755 index 0000000000..33a08b69e2 --- /dev/null +++ b/scripts/fetch-brew-bottle.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -eof pipefail + +# This script is used to fetch HomeBrew bottles for PCRE and OpenSSL. + +function get_gh_pkgs_token() { + curl --fail -Ls -u "${GITHUB_USER}:${GITHUB_TOKEN}" https://ghcr.io/token | jq -r '.token' +} + +function get_bottle_json() { + brew info --json=v1 "${1}" | jq '.[0].bottle.stable.files.mojave' +} + +function fetch_bottle() { + if [[ -n "${BEARER_TOKEN}" ]]; then + AUTH=("-H" "Authorization: Bearer ${BEARER_TOKEN}") + else + AUTH=("-u" "_:_") # WARNING: Unauthorized requests can be throttled. + fi + curl --fail -Ls "${AUTH[@]}" -o "${1}" "${2}" +} + +if [[ $(uname) != "Darwin" ]]; then + echo "This script is intended for use on MacOS!" >&2 + exit 1 +fi + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 1 +fi +BOTTLE_NAME="${1}" +BOTTLE_PATH="/tmp/${BOTTLE_NAME}.tar.gz" + +# GitHub Packages requires authentication. +GITHUB_USER="${GITHUB_USER:-_}" +GITHUB_TOKEN="${GITHUB_TOKEN:-_}" +if [[ "${GITHUB_USER}" == "_" ]] || [[ "${GITHUB_TOKEN}" == "_" ]]; then + echo "No GITHUB_USER or GITHUB_TOKEN variable set!" >&2 + echo "GitHub Packages which can throttle unauthorized requests." >&2 +else + echo "${BOTTLE_NAME} - Fetching GH Pkgs Token" + BEARER_TOKEN=$(get_gh_pkgs_token) +fi + +# We want the most recent available version of the package. +if [[ $(stat -f %u /usr/local/var/homebrew) -ne "${UID}" ]]; then + echo "Missing permissions to update Homebrew formulae!" >&2 +else + echo "${BOTTLE_NAME} - Updateing HomeBrew repository" + brew update >/dev/null +fi + +echo "${BOTTLE_NAME} - Finding bottle URL" +BOTTLE_JSON=$(get_bottle_json "${BOTTLE_NAME}") +BOTTLE_URL=$(echo "${BOTTLE_JSON}" | jq -r .url) +BOTTLE_SHA=$(echo "${BOTTLE_JSON}" | jq -r .sha256) + +echo "${BOTTLE_NAME} - Fetching bottles for macOS" +fetch_bottle "${BOTTLE_PATH}" "${BOTTLE_URL}" +trap "rm -fr ${BOTTLE_PATH}" EXIT ERR INT QUIT + +echo "${BOTTLE_NAME} - Checking SHA256 checksum" +BOTTLE_LOCAL_SHA=$(shasum -a 256 "${BOTTLE_PATH}" | awk '{print $1}') + +if [[ "${BOTTLE_LOCAL_SHA}" != "${BOTTLE_SHA}" ]]; then + echo "The SHA256 of downloaded bottle did not match!" >&2 + exit 1; +fi + +echo "${BOTTLE_NAME} - Unpacking bottle tarball" +mkdir -p "bottles/${BOTTLE_NAME}" +tar xzf "${BOTTLE_PATH}" --strip-components 2 -C "bottles/${BOTTLE_NAME}"