From 2543413017f8fb634025fc71d9b5b475053bec28 Mon Sep 17 00:00:00 2001 From: Radu Tutueanu Date: Wed, 23 Nov 2016 15:09:47 +0100 Subject: [PATCH] Build and publish node pre-gyp binaries (#657) * Build node pre-gyp for linux * Add Jenkisfile * Fix core package names * Fix Jenkinsfile errors * To revert: only build using 4.4.7 * Do not download on linux * Fix wrong argument * Fix archiving linux artifacts * Try to fix * Revert "Try to fix" This reverts commit 8bcd1d0ffb539b9f2881ce8e0396681b55caf69e. * Fix moving the gyp binaries * Update version * Revert "To revert: only build using 4.4.7" This reverts commit b829c0e0d27b1baed50c3af436731c5014de290f. * Bump version in dependencies * Publish binaries * Use correct version * Remove unneded quote * Fix publish --- dependencies.list | 2 +- packaging/.gitignore | 3 + packaging/Jenkinsfile.node-pre-gyp | 140 ++++++++++++++++++ packaging/functions.sh | 100 +++++++++++++ packaging/node-pre-gyp/base-image/Dockerfile | 37 +++++ packaging/node-pre-gyp/build-image/Dockerfile | 4 + .../build-image/inside/build-package | 34 +++++ .../build-image/inside/docker-entrypoint.sh | 3 + packaging/node-pre-gyp/test-image/Dockerfile | 3 + packaging/node-pre-gyp/test-image/inside/test | 14 ++ packaging/package.sh | 80 ++++++++++ scripts/build-node-pre-gyp.sh | 2 +- 12 files changed, 420 insertions(+), 2 deletions(-) create mode 100644 packaging/.gitignore create mode 100644 packaging/Jenkinsfile.node-pre-gyp create mode 100755 packaging/functions.sh create mode 100644 packaging/node-pre-gyp/base-image/Dockerfile create mode 100644 packaging/node-pre-gyp/build-image/Dockerfile create mode 100755 packaging/node-pre-gyp/build-image/inside/build-package create mode 100755 packaging/node-pre-gyp/build-image/inside/docker-entrypoint.sh create mode 100644 packaging/node-pre-gyp/test-image/Dockerfile create mode 100755 packaging/node-pre-gyp/test-image/inside/test create mode 100755 packaging/package.sh diff --git a/dependencies.list b/dependencies.list index 6e4c7a6b..c6e9e123 100644 --- a/dependencies.list +++ b/dependencies.list @@ -1,5 +1,5 @@ PACKAGE_NAME=realm-js -VERSION=0.14.3-6 +VERSION=0.15.1-rc REALM_CORE_VERSION=2.1.4 REALM_SYNC_VERSION=1.0.0-BETA-3.3 REALM_OBJECT_SERVER_VERSION=1.0.0-BETA-2.1 diff --git a/packaging/.gitignore b/packaging/.gitignore new file mode 100644 index 00000000..28a0d4d0 --- /dev/null +++ b/packaging/.gitignore @@ -0,0 +1,3 @@ +/out +/test-logs + diff --git a/packaging/Jenkinsfile.node-pre-gyp b/packaging/Jenkinsfile.node-pre-gyp new file mode 100644 index 00000000..3bf75eec --- /dev/null +++ b/packaging/Jenkinsfile.node-pre-gyp @@ -0,0 +1,140 @@ +#!groovy + +// parameter: PUBLISH +repoName = 'realm-js' // This is a global variable + +def setBuildName(newBuildName) { + currentBuild.displayName = "${currentBuild.displayName} - ${newBuildName}" +} + +def unstashSources() { + sshagent(['realm-ci-ssh']) { + sh 'rm -rf * .git' + checkout scm + sh 'git clean -ffdx -e .????????' + sh 'git submodule update --init --recursive' + } +} + +def readGitTag() { + sh "git describe --exact-match --tags HEAD | tail -n 1 > tag.txt 2>&1 || true" + def tag = readFile('tag.txt').trim() + return tag +} + +def readGitSha() { + sh "git rev-parse HEAD | cut -b1-8 > sha.txt" + def sha = readFile('sha.txt').readLines().last().trim() + return sha +} + +def getVersion(){ + def dependencies = readProperties file: 'dependencies.list' + def gitTag = readGitTag() + def gitSha = readGitSha() + if (gitTag == "") { + return "${dependencies.VERSION}-g${gitSha}" + } + else { + return dependencies.VERSION + } +} + +def doBuildLinux(dependencies) { + return { + node('docker') { + unstashSources() + + docker.withRegistry("https://${env.DOCKER_REGISTRY}", "ecr:eu-west-1:aws-ci-user") { + withCredentials([[$class: 'StringBinding', credentialsId: 'packagecloud-sync-devel-master-token', variable: 'PACKAGECLOUD_MASTER_TOKEN']]) { + withEnv(["EXTRA_NPM_ARGUMENTS=--realm_download_binaries=0"]) { + sh 'sh packaging/package.sh node-pre-gyp' + } + } + } + + dir('packaging/out/node-pre-gyp') { + stash includes: "realm-*", name: "compiled-linux" + archiveArtifacts "realm-*" + } + } + } +} + +def doBuildMac(dependencies) { + return { + node('osx_vegas') { + unstashSources() + sh ''' + ./scripts/build-node-pre-gyp.sh + tar cvfz darwin-compiled.tgz compiled/ + ''' + dir('out') { + stash includes: 'realm-*', name: "compiled-mac" + archiveArtifacts "realm-*" + } + + } + } +} + +def doBuild(dependencies) { + parallel( + "build_mac": doBuildMac(dependencies), + "build_linux": doBuildLinux(dependencies) + ) +} + +def doPublish() { + node('aws') { + sh 'rm -rf *' + unstash 'compiled-mac' + unstash 'compiled-linux' + withCredentials([[$class: 'FileBinding', credentialsId: 'c0cc8f9e-c3f1-4e22-b22f-6568392e26ae', variable: 's3cfg_config_file']]) { + sh """ + s3cmd -c \$s3cfg_config_file put realm-* 's3://static.realm.io/node-pre-gyp/' + rm -rf * + """ + } + } +} + +def gitTag +def gitSha +def dependencies +def version + +stage 'check' +node('docker') { + unstashSources() + dependencies = readProperties file: 'dependencies.list' + echo "VERSION: ${dependencies.VERSION}" + echo "NODEJS_REALM_VERSION: ${dependencies.NODEJS_REALM_VERSION}" + echo "NODEJS_REALM_SERVER_VERSION: ${dependencies.NODEJS_REALM_SERVER_VERSION}" + + gitTag = readGitTag() + gitSha = readGitSha() + version = getVersion() + echo "tag: ${gitTag}" + if (gitTag == "") { + echo "No tag given for this build" + setBuildName("${gitSha}") + } else { + if (gitTag != "v${dependencies.VERSION}") { + echo "Git tag '${gitTag}' does not match v${dependencies.VERSION}" + } else { + echo "Building release: '${gitTag}'" + setBuildName("Tag ${gitTag}") + } + } + + echo "version: ${version}" +} + +stage 'build' +doBuild(dependencies) + +if (PUBLISH == 'true') { + stage 'publish' + doPublish() +} diff --git a/packaging/functions.sh b/packaging/functions.sh new file mode 100755 index 00000000..6be864e0 --- /dev/null +++ b/packaging/functions.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +die() { echo "$@" 1>&2 ; exit 1; } +info() { echo "===> $*"; } +apply_shell_expansion() { + declare file="$1" + declare data=$(< "$file") + declare delimiter="__apply_shell_expansion_delimiter__" + declare command="cat <<$delimiter"$'\n'"$data"$'\n'"$delimiter" + eval "$command" +} + +docker_build() { + declare name="$1"; shift + declare path="$1"; shift + declare args="$*" + + if [ "${DOCKER_REGISTRY}" != "" ]; then + remote_name="${DOCKER_REGISTRY}/${name}" + fi + + info "Building ${name} image..." + if [ "${DOCKER_REGISTRY}" != "" ]; then + docker_pull "${remote_name}" && docker tag "${remote_name}" "${name}" || true + fi + + old_id=$(docker images -q "${name}") + info "Old ${name} image id: ${old_id}" + + if [ "${DOCKERFILE}" != "" ]; then + docker build ${args} -t "${name}" -f "${DOCKERFILE}" "${path}" || \ + die "Building ${name} image failed" + else + docker build ${args} -t "${name}" "${path}" || \ + die "Building ${name} image failed" + fi + + new_id=$(docker images -q "${name}") + info "New ${name} image id: ${new_id}" + + if [ "${DEBUG}" ] && [ "${new_id}" != "${old_id}" ]; then + info "History for old id $old_id:" + if [ "${old_id}" != "" ]; then + docker history "$old_id" + fi + + info "History for new id $new_id:" + docker history "$new_id" + fi + + if [ "${NOPUSH:-0}" != "1" ] && [ "${DOCKER_REGISTRY}" != "" ]; then + docker tag "${name}" "${remote_name}" + docker_push "${remote_name}" + fi +} + +# Due to https://github.com/docker/docker/issues/20316, we use +# https://github.com/tonistiigi/buildcache to generate a cache of the layer +# metadata for later builds. +my_buildcache() { + if [ "$DOCKER_REGISTRY" != "" ]; then + + docker_path="/var/lib/docker" + + # Stupid hack for our AWS nodes, which have docker data on another volume + if [ -d "/mnt/docker" ]; then + docker_path="/mnt/docker" + fi + + docker pull "${DOCKER_REGISTRY}/ci/buildcache" >/dev/null && \ + docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ${docker_path}:/var/lib/docker \ + "${DOCKER_REGISTRY}/ci/buildcache" "$@" + fi +} + +docker_pull() { + info "Attempting to pull '$1' image from registry..." + ( + tmpdir=$(mktemp -d) + docker pull "$1" + ( + # In addition, pull the cached metadata and load it (buildcache) + cd "${tmpdir}" && docker pull "$1-cache" && docker save "$1-cache" | \ + tar -xf - && docker load -i ./*/layer.tar + ) + rm -rf "${tmpdir}" + ) || true +} + +docker_push() { + info "Pushing '$1' image to registry..." + docker push "$1" + # Create a cache of the layer metdata we need and push it as an image + #docker rmi $1-cache 2>/dev/null || true + my_buildcache save -g /var/lib/docker "$1" | gunzip -c | \ + docker import - "$1-cache" && \ + docker push "$1-cache" +} diff --git a/packaging/node-pre-gyp/base-image/Dockerfile b/packaging/node-pre-gyp/base-image/Dockerfile new file mode 100644 index 00000000..03bf127c --- /dev/null +++ b/packaging/node-pre-gyp/base-image/Dockerfile @@ -0,0 +1,37 @@ +FROM alanfranz/fwd-centos-6:latest + +ARG PACKAGECLOUD_URL + +ENV NPM_CONFIG_UNSAFE_PERM true + +# Install EPEL & devtoolset +# On CentOS6, there is a bug with OverlayFS and Docker. It is needed to touch +# /var/lib/rpm/* in order to work around this issue. +# Link: https://github.com/docker/docker/issues/10180 + +RUN touch /var/lib/rpm/* \ + && yum -y install \ + epel-release \ + centos-release-scl-rh \ + && yum-config-manager --enable rhel-server-rhscl-6-rpms \ + && curl -s $PACKAGECLOUD_URL/script.rpm.sh | bash \ + && yum -y install \ + python27 \ + which \ + chrpath \ + openssl-devel \ + devtoolset-3-gcc \ + devtoolset-3-gcc-c++ \ + devtoolset-3-binutils \ + libconfig-devel \ + jq \ + && yum remove -y g++ gcc \ + && yum clean all + +ENV NVM_DIR /root/.nvm +RUN wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.31.4/install.sh | bash +RUN . $NVM_DIR/nvm.sh && \ + nvm install 4.4.7 && \ + nvm install 5.12.0 && \ + nvm install 6.5.0 && \ + nvm install 7.0.0 diff --git a/packaging/node-pre-gyp/build-image/Dockerfile b/packaging/node-pre-gyp/build-image/Dockerfile new file mode 100644 index 00000000..7ea13bf6 --- /dev/null +++ b/packaging/node-pre-gyp/build-image/Dockerfile @@ -0,0 +1,4 @@ +FROM ci/realm-js:node-pre-gyp-base + +ENTRYPOINT ["/bin/bash", "/inside/docker-entrypoint.sh"] +CMD ["/bin/bash", "/inside/build-package"] diff --git a/packaging/node-pre-gyp/build-image/inside/build-package b/packaging/node-pre-gyp/build-image/inside/build-package new file mode 100755 index 00000000..aee4ef23 --- /dev/null +++ b/packaging/node-pre-gyp/build-image/inside/build-package @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +# we should always set proper ownership before exiting, otherwise +# the created packages will have root:root ownership and we'll be unable +# to delete them from our host. +trap 'chown -R --reference /inside/build-package /out/' EXIT + +. $NVM_DIR/nvm.sh + +REALM_CORE_VERSION=$(echo ${REALM_CORE_VERSION} | sed 's/-\([^g]\)/_\1/g') + +yum install -y \ + realm-devel-${REALM_CORE_VERSION} \ + realm-node-devel-${REALM_CORE_VERSION} + +export NPM_CONFIG_UNSAFE_PERM=true # because we're running as root + +setup_source() { + path=$1 + cp -a /source $path + cd $path + json=$(jq ".version = \"${VERSION}\"" package.json) + echo $json > package.json + rm -rf compiled node_modules .nvm +} + +setup_source "/tmp/build" +./scripts/build-node-pre-gyp.sh + +nvm use 4.4.7 + +cp out/* /out/ diff --git a/packaging/node-pre-gyp/build-image/inside/docker-entrypoint.sh b/packaging/node-pre-gyp/build-image/inside/docker-entrypoint.sh new file mode 100755 index 00000000..3d0ece23 --- /dev/null +++ b/packaging/node-pre-gyp/build-image/inside/docker-entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +scl enable devtoolset-3 -- "$@" diff --git a/packaging/node-pre-gyp/test-image/Dockerfile b/packaging/node-pre-gyp/test-image/Dockerfile new file mode 100644 index 00000000..3b4c58db --- /dev/null +++ b/packaging/node-pre-gyp/test-image/Dockerfile @@ -0,0 +1,3 @@ +FROM ci/realm-js:node-pre-gyp-base + +CMD ["/bin/bash", "/inside/test"] diff --git a/packaging/node-pre-gyp/test-image/inside/test b/packaging/node-pre-gyp/test-image/inside/test new file mode 100755 index 00000000..0f546bfe --- /dev/null +++ b/packaging/node-pre-gyp/test-image/inside/test @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# we should always set proper ownership before exiting, otherwise +# the created logs will have root:root ownership and we'll be unable +# to delete them from our host. +trap 'chown -R --reference $0 /test-logs/' EXIT + +. $NVM_DIR/nvm.sh + +ls -la /out + +nvm use 4.4.7 diff --git a/packaging/package.sh b/packaging/package.sh new file mode 100755 index 00000000..9c307c7a --- /dev/null +++ b/packaging/package.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +set -e + +script_path="$(pushd "$(dirname "$0")" >/dev/null; pwd)" +src_path="$(pushd "${script_path}/.." >/dev/null; pwd)" + +. "${script_path}/functions.sh" + + +git_tag=$(git describe --exact-match --tags HEAD 2>/dev/null || true) + +. "${src_path}/dependencies.list" + +export PACKAGECLOUD_URL="https://${PACKAGECLOUD_MASTER_TOKEN}:@packagecloud.io/install/repositories/realm/sync-devel" + +if [ -z "$git_tag" ]; then + info "No git tag exists. Triggering -devel build" + sha=$(git rev-parse HEAD | cut -b1-8) + ITERATION="0.$sha" +elif [ "$git_tag" != "v${VERSION}" ]; then + die "Git tag '$git_tag' does not match VERSION: '$VERSION'" +else + info "Found git tag: '$git_tag'. Triggering release build" + ITERATION="${BUILD_NUMBER:-1}" +fi + +rm -rf "${src_path}/packaging/out"; mkdir -p "${src_path}/packaging/out" + +cp "${src_path}/dependencies.list" "${src_path}/packaging/out/packaging.list" +cat <<-EOD >> "${src_path}/packaging/out/packaging.list" +ITERATION=$ITERATION +EXTRA_NPM_ARGUMENTS=$EXTRA_NPM_ARGUMENTS +EOD + +env_file="${src_path}/packaging/out/packaging.list" + +default="centos-6 centos-7 ubuntu-1604" +for distro in ${*:-$default}; do + distro_path="${src_path}/packaging/${distro}" + image_name="ci/${PACKAGE_NAME}:${distro}" + + mkdir -p "${src_path}/packaging/out/$distro" + rm -f "${src_path}/packaging/out/$distro/*" + + mkdir -p "${src_path}/packaging/test-logs/$distro" + rm -f "${src_path}/packaging/test-logs/$distro/*" + + docker_build "${image_name}-base" "${distro_path}/base-image" \ + --build-arg "PACKAGECLOUD_URL=$PACKAGECLOUD_URL" + + docker_build "${image_name}-build" "${distro_path}/build-image" + + info "Running '$distro' build..." + docker run \ + --env-file "${env_file}" \ + --rm \ + -v "${src_path}/packaging/${distro}/files:/files:ro,z" \ + -v "${src_path}/packaging/${distro}/build-image/inside:/inside:ro,z" \ + -v "${src_path}:/source:ro,z" \ + -v "${src_path}/packaging/common:/common:ro,z" \ + -v "${src_path}/packaging/out/${distro}:/out:z" \ + -w /inside "${image_name}-build" \ + || die "Build phase for '$distro' failed." + + docker_build "${image_name}-test" "${distro_path}/test-image" + + info "Running '$distro' tests..." + docker run \ + --env-file "${env_file}" \ + --rm \ + -v "${src_path}/packaging/${distro}/test-image/inside:/inside:ro,z" \ + -v "${src_path}/packaging/out/${distro}:/out:z" \ + -v "${src_path}/packaging/test-logs/${distro}:/test-logs:z" \ + -v "${src_path}/packaging/common:/common:ro,z" \ + -w /inside "${image_name}-test" \ + || die "Test phase for '$distro' failed." + + info "Test phase for '$distro' succeeded." +done diff --git a/scripts/build-node-pre-gyp.sh b/scripts/build-node-pre-gyp.sh index ba0998b7..bd7bdd2e 100755 --- a/scripts/build-node-pre-gyp.sh +++ b/scripts/build-node-pre-gyp.sh @@ -1,6 +1,6 @@ #!/bin/bash -node_versions=${@:-4.4.7 5.12.0 6.5.0} +node_versions=${@:-4.4.7 5.12.0 6.5.0 7.0.0} topdir=$(cd $(dirname "$0")/..; pwd)