Build and publish node pre-gyp binaries (#657)

* Build node pre-gyp for linux

* Add Jenkisfile

* Fix core package names

* Fix Jenkinsfile errors

* To revert: only build using 4.4.7

* Do not download on linux

* Fix wrong argument

* Fix archiving linux artifacts

* Try to fix

* Revert "Try to fix"

This reverts commit 8bcd1d0ffb539b9f2881ce8e0396681b55caf69e.

* Fix moving the gyp binaries

* Update version

* Revert "To revert: only build using 4.4.7"

This reverts commit b829c0e0d27b1baed50c3af436731c5014de290f.

* Bump version in dependencies

* Publish binaries

* Use correct version

* Remove unneded quote

* Fix publish

dependencies.list

@@ -1,5 +1,5 @@
 PACKAGE_NAME=realm-js
-VERSION=0.14.3-6
+VERSION=0.15.1-rc
 REALM_CORE_VERSION=2.1.4
 REALM_SYNC_VERSION=1.0.0-BETA-3.3
 REALM_OBJECT_SERVER_VERSION=1.0.0-BETA-2.1

packaging/.gitignore

@@ -0,0 +1,3 @@
+/out
+/test-logs
+

packaging/Jenkinsfile.node-pre-gyp

@@ -0,0 +1,140 @@
+#!groovy
+
+// parameter: PUBLISH
+repoName = 'realm-js' // This is a global variable
+
+def setBuildName(newBuildName) {
+  currentBuild.displayName = "${currentBuild.displayName} - ${newBuildName}"
+}
+
+def unstashSources() {
+  sshagent(['realm-ci-ssh']) {
+    sh 'rm -rf * .git'
+    checkout scm
+    sh 'git clean -ffdx -e .????????'
+    sh 'git submodule update --init --recursive'
+  }
+}
+
+def readGitTag() {
+  sh "git describe --exact-match --tags HEAD | tail -n 1 > tag.txt 2>&1 || true"
+  def tag = readFile('tag.txt').trim()
+  return tag
+}
+
+def readGitSha() {
+  sh "git rev-parse HEAD | cut -b1-8 > sha.txt"
+  def sha = readFile('sha.txt').readLines().last().trim()
+  return sha
+}
+
+def getVersion(){
+  def dependencies = readProperties file: 'dependencies.list'
+  def gitTag = readGitTag()
+  def gitSha = readGitSha()
+  if (gitTag == "") {
+    return "${dependencies.VERSION}-g${gitSha}"
+  }
+  else {
+    return dependencies.VERSION
+  }
+}
+
+def doBuildLinux(dependencies) {
+  return {
+    node('docker') {
+      unstashSources()
+
+      docker.withRegistry("https://${env.DOCKER_REGISTRY}", "ecr:eu-west-1:aws-ci-user") {
+        withCredentials([[$class: 'StringBinding', credentialsId: 'packagecloud-sync-devel-master-token', variable: 'PACKAGECLOUD_MASTER_TOKEN']]) {
+          withEnv(["EXTRA_NPM_ARGUMENTS=--realm_download_binaries=0"]) {
+            sh 'sh packaging/package.sh node-pre-gyp'
+          }
+        }
+      }
+
+      dir('packaging/out/node-pre-gyp') {
+        stash includes: "realm-*", name: "compiled-linux"
+        archiveArtifacts "realm-*"
+      }
+    }
+  }
+}
+
+def doBuildMac(dependencies) {
+  return {
+    node('osx_vegas') {
+      unstashSources()
+          sh '''
+            ./scripts/build-node-pre-gyp.sh
+            tar cvfz darwin-compiled.tgz compiled/
+          '''
+      dir('out') {
+        stash includes: 'realm-*', name: "compiled-mac"
+        archiveArtifacts "realm-*"
+      }
+
+    }
+  }
+}
+
+def doBuild(dependencies) {
+    parallel(
+      "build_mac": doBuildMac(dependencies),
+      "build_linux": doBuildLinux(dependencies)
+    )
+}
+
+def doPublish() {
+  node('aws') {
+    sh 'rm -rf *'
+    unstash 'compiled-mac'
+    unstash 'compiled-linux'
+    withCredentials([[$class: 'FileBinding', credentialsId: 'c0cc8f9e-c3f1-4e22-b22f-6568392e26ae', variable: 's3cfg_config_file']]) {
+      sh """
+        s3cmd -c \$s3cfg_config_file put realm-* 's3://static.realm.io/node-pre-gyp/'
+        rm -rf *
+      """
+    }
+  }
+}
+
+def gitTag
+def gitSha
+def dependencies
+def version
+
+stage 'check'
+node('docker') {
+  unstashSources()
+  dependencies = readProperties file: 'dependencies.list'
+  echo "VERSION: ${dependencies.VERSION}"
+  echo "NODEJS_REALM_VERSION: ${dependencies.NODEJS_REALM_VERSION}"
+  echo "NODEJS_REALM_SERVER_VERSION: ${dependencies.NODEJS_REALM_SERVER_VERSION}"
+
+  gitTag = readGitTag()
+  gitSha = readGitSha()
+  version = getVersion()
+  echo "tag: ${gitTag}"
+  if (gitTag == "") {
+    echo "No tag given for this build"
+    setBuildName("${gitSha}")
+  } else {
+    if (gitTag != "v${dependencies.VERSION}") {
+      echo "Git tag '${gitTag}' does not match v${dependencies.VERSION}"
+    } else {
+      echo "Building release: '${gitTag}'"
+      setBuildName("Tag ${gitTag}")
+    }
+  }
+
+  echo "version: ${version}"
+}
+
+stage 'build'
+doBuild(dependencies)
+
+if (PUBLISH == 'true') {
+  stage 'publish'
+  doPublish()
+}

packaging/functions.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+die() { echo "$@" 1>&2 ; exit 1; }
+info() { echo "===> $*"; }
+apply_shell_expansion() {
+  declare file="$1"
+  declare data=$(< "$file")
+  declare delimiter="__apply_shell_expansion_delimiter__"
+  declare command="cat <<$delimiter"$'\n'"$data"$'\n'"$delimiter"
+  eval "$command"
+}
+
+docker_build() {
+  declare name="$1"; shift
+  declare path="$1"; shift
+  declare args="$*"
+
+  if [ "${DOCKER_REGISTRY}" != "" ]; then
+    remote_name="${DOCKER_REGISTRY}/${name}"
+  fi
+
+  info "Building ${name} image..."
+  if [ "${DOCKER_REGISTRY}" != "" ]; then
+    docker_pull "${remote_name}" && docker tag "${remote_name}" "${name}" || true
+  fi
+
+  old_id=$(docker images -q "${name}")
+  info "Old ${name} image id: ${old_id}"
+  
+  if [ "${DOCKERFILE}" != "" ]; then
+    docker build ${args} -t "${name}" -f "${DOCKERFILE}" "${path}" || \
+        die "Building ${name} image failed"
+  else
+    docker build ${args} -t "${name}" "${path}" || \
+        die "Building ${name} image failed"
+  fi
+
+  new_id=$(docker images -q "${name}")
+  info "New ${name} image id: ${new_id}"
+
+  if [ "${DEBUG}" ] && [ "${new_id}" != "${old_id}" ]; then
+    info "History for old id $old_id:"
+    if [ "${old_id}" != "" ]; then
+      docker history "$old_id"
+    fi
+
+    info "History for new id $new_id:"
+    docker history "$new_id"
+  fi
+
+  if [ "${NOPUSH:-0}" != "1" ] && [ "${DOCKER_REGISTRY}" != "" ]; then
+    docker tag "${name}" "${remote_name}"
+    docker_push "${remote_name}"
+  fi
+}
+
+# Due to https://github.com/docker/docker/issues/20316, we use
+# https://github.com/tonistiigi/buildcache to generate a cache of the layer
+# metadata for later builds.
+my_buildcache() {
+  if [ "$DOCKER_REGISTRY" != "" ]; then
+
+    docker_path="/var/lib/docker"
+
+    # Stupid hack for our AWS nodes, which have docker data on another volume
+    if [ -d "/mnt/docker" ]; then
+      docker_path="/mnt/docker"
+    fi
+
+    docker pull "${DOCKER_REGISTRY}/ci/buildcache" >/dev/null && \
+    docker run --rm \
+      -v /var/run/docker.sock:/var/run/docker.sock \
+      -v ${docker_path}:/var/lib/docker \
+      "${DOCKER_REGISTRY}/ci/buildcache" "$@"
+  fi
+}
+
+docker_pull() {
+  info "Attempting to pull '$1' image from registry..."
+  (
+    tmpdir=$(mktemp -d)
+    docker pull "$1"
+    (
+      # In addition, pull the cached metadata and load it (buildcache)
+      cd "${tmpdir}" && docker pull "$1-cache" && docker save "$1-cache" | \
+        tar -xf -  && docker load -i ./*/layer.tar
+    )
+    rm -rf "${tmpdir}"
+  ) || true
+}
+
+docker_push() {
+  info "Pushing '$1' image to registry..."
+  docker push "$1"
+  # Create a cache of the layer metdata we need and push it as an image
+  #docker rmi $1-cache 2>/dev/null || true
+  my_buildcache save -g /var/lib/docker "$1" | gunzip -c | \
+    docker import - "$1-cache" && \
+    docker push "$1-cache"
+}

packaging/node-pre-gyp/base-image/Dockerfile

@@ -0,0 +1,37 @@
+FROM alanfranz/fwd-centos-6:latest
+
+ARG PACKAGECLOUD_URL
+
+ENV NPM_CONFIG_UNSAFE_PERM true
+
+# Install EPEL & devtoolset
+# On CentOS6, there is a bug with OverlayFS and Docker. It is needed to touch
+# /var/lib/rpm/* in order to work around this issue.
+# Link: https://github.com/docker/docker/issues/10180
+
+RUN touch /var/lib/rpm/* \
+    && yum -y install \
+        epel-release \
+        centos-release-scl-rh \
+    && yum-config-manager --enable rhel-server-rhscl-6-rpms \
+    && curl -s $PACKAGECLOUD_URL/script.rpm.sh | bash \
+    && yum -y install \
+        python27 \
+        which \
+        chrpath \
+        openssl-devel \
+        devtoolset-3-gcc \
+        devtoolset-3-gcc-c++ \
+        devtoolset-3-binutils \
+        libconfig-devel \
+        jq \
+    && yum remove -y g++ gcc \
+    && yum clean all
+
+ENV NVM_DIR /root/.nvm
+RUN wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.31.4/install.sh | bash
+RUN . $NVM_DIR/nvm.sh && \
+    nvm install 4.4.7 && \
+    nvm install 5.12.0 && \
+    nvm install 6.5.0 && \
+    nvm install 7.0.0

packaging/node-pre-gyp/build-image/Dockerfile

@@ -0,0 +1,4 @@
+FROM ci/realm-js:node-pre-gyp-base
+
+ENTRYPOINT ["/bin/bash", "/inside/docker-entrypoint.sh"]
+CMD ["/bin/bash", "/inside/build-package"]

packaging/node-pre-gyp/build-image/inside/build-package

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+# we should always set proper ownership before exiting, otherwise
+# the created packages will have root:root ownership and we'll be unable
+# to delete them from our host.
+trap 'chown -R --reference /inside/build-package /out/' EXIT
+
+. $NVM_DIR/nvm.sh
+
+REALM_CORE_VERSION=$(echo ${REALM_CORE_VERSION} | sed 's/-\([^g]\)/_\1/g')
+
+yum install -y \
+  realm-devel-${REALM_CORE_VERSION} \
+  realm-node-devel-${REALM_CORE_VERSION}
+
+export NPM_CONFIG_UNSAFE_PERM=true # because we're running as root
+
+setup_source() {
+  path=$1
+  cp -a /source $path
+  cd $path
+  json=$(jq ".version = \"${VERSION}\"" package.json)
+  echo $json > package.json
+  rm -rf compiled node_modules .nvm
+}
+
+setup_source "/tmp/build"
+./scripts/build-node-pre-gyp.sh
+
+nvm use 4.4.7
+
+cp out/* /out/

packaging/node-pre-gyp/build-image/inside/docker-entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+scl enable devtoolset-3 -- "$@"

packaging/node-pre-gyp/test-image/Dockerfile

@@ -0,0 +1,3 @@
+FROM ci/realm-js:node-pre-gyp-base
+
+CMD ["/bin/bash", "/inside/test"]

packaging/node-pre-gyp/test-image/inside/test

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# we should always set proper ownership before exiting, otherwise
+# the created logs will have root:root ownership and we'll be unable
+# to delete them from our host.
+trap 'chown -R --reference $0 /test-logs/' EXIT
+
+. $NVM_DIR/nvm.sh
+
+ls -la /out
+
+nvm use 4.4.7

packaging/package.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -e
+
+script_path="$(pushd "$(dirname "$0")" >/dev/null; pwd)"
+src_path="$(pushd "${script_path}/.." >/dev/null; pwd)"
+
+. "${script_path}/functions.sh"
+
+
+git_tag=$(git describe --exact-match --tags HEAD 2>/dev/null || true)
+
+. "${src_path}/dependencies.list"
+
+export PACKAGECLOUD_URL="https://${PACKAGECLOUD_MASTER_TOKEN}:@packagecloud.io/install/repositories/realm/sync-devel"
+
+if [ -z "$git_tag" ]; then
+  info "No git tag exists.  Triggering -devel build"
+  sha=$(git rev-parse HEAD | cut -b1-8)
+  ITERATION="0.$sha"
+elif [ "$git_tag" != "v${VERSION}" ]; then
+  die "Git tag '$git_tag' does not match VERSION: '$VERSION'"
+else
+  info "Found git tag: '$git_tag'. Triggering release build"
+  ITERATION="${BUILD_NUMBER:-1}"
+fi
+
+rm -rf "${src_path}/packaging/out"; mkdir -p "${src_path}/packaging/out"
+
+cp "${src_path}/dependencies.list" "${src_path}/packaging/out/packaging.list"
+cat <<-EOD >> "${src_path}/packaging/out/packaging.list"
+ITERATION=$ITERATION
+EXTRA_NPM_ARGUMENTS=$EXTRA_NPM_ARGUMENTS
+EOD
+
+env_file="${src_path}/packaging/out/packaging.list"
+
+default="centos-6 centos-7 ubuntu-1604"
+for distro in ${*:-$default}; do
+  distro_path="${src_path}/packaging/${distro}"
+  image_name="ci/${PACKAGE_NAME}:${distro}"
+
+  mkdir -p "${src_path}/packaging/out/$distro"
+  rm -f "${src_path}/packaging/out/$distro/*"
+
+  mkdir -p "${src_path}/packaging/test-logs/$distro"
+  rm -f "${src_path}/packaging/test-logs/$distro/*"
+
+  docker_build "${image_name}-base" "${distro_path}/base-image" \
+      --build-arg "PACKAGECLOUD_URL=$PACKAGECLOUD_URL"
+
+  docker_build "${image_name}-build" "${distro_path}/build-image" 
+
+  info "Running '$distro' build..."
+  docker run \
+    --env-file "${env_file}" \
+    --rm \
+    -v "${src_path}/packaging/${distro}/files:/files:ro,z" \
+    -v "${src_path}/packaging/${distro}/build-image/inside:/inside:ro,z" \
+    -v "${src_path}:/source:ro,z" \
+    -v "${src_path}/packaging/common:/common:ro,z" \
+    -v "${src_path}/packaging/out/${distro}:/out:z" \
+    -w /inside "${image_name}-build" \
+    || die "Build phase for '$distro' failed."
+
+  docker_build "${image_name}-test" "${distro_path}/test-image"
+
+  info "Running '$distro' tests..."
+  docker run \
+      --env-file "${env_file}" \
+      --rm \
+      -v "${src_path}/packaging/${distro}/test-image/inside:/inside:ro,z" \
+      -v "${src_path}/packaging/out/${distro}:/out:z" \
+      -v "${src_path}/packaging/test-logs/${distro}:/test-logs:z" \
+      -v "${src_path}/packaging/common:/common:ro,z" \
+      -w /inside "${image_name}-test" \
+      || die "Test phase for '$distro' failed."
+
+  info "Test phase for '$distro' succeeded."
+done

scripts/build-node-pre-gyp.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-node_versions=${@:-4.4.7 5.12.0 6.5.0}
+node_versions=${@:-4.4.7 5.12.0 6.5.0 7.0.0}
 
 topdir=$(cd $(dirname "$0")/..; pwd)