From 410db7da3c8ffa8b3523ad4b72d492b0c7b61191 Mon Sep 17 00:00:00 2001 From: rh389 Date: Tue, 24 Oct 2017 02:26:06 +0100 Subject: [PATCH 1/3] Add a .manifest.list to target. Skip download/extract if target manifest matches. --- scripts/download-realm.js | 168 ++++++++++++++++++++++++-------------- 1 file changed, 107 insertions(+), 61 deletions(-) diff --git a/scripts/download-realm.js b/scripts/download-realm.js index c50564a4..0361ce86 100644 --- a/scripts/download-realm.js +++ b/scripts/download-realm.js @@ -23,9 +23,11 @@ const path = require('path'); const os = require('os'); const child_process = require('child_process'); const fetch = require('node-fetch'); -const ini = require('ini').parse; +const ini = require('ini'); const decompress = require('decompress'); +const MANIFEST_FILENAME = '.manifest.list'; + function exec() { const args = Array.from(arguments); return new Promise((resolve, reject) => { @@ -112,37 +114,16 @@ function extract(downloadedArchive, targetFolder, archiveRootFolder) { } } -function acquireCore(options, dependencies, target) { - let serverFolder = `core/v${dependencies.REALM_CORE_VERSION}`; - let flavor = options.debug ? 'Debug' : 'Release'; +function acquire(desired, target) { + const corePath = desired.CORE_ARCHIVE && path.resolve(os.tmpdir(), desired.CORE_ARCHIVE); + const syncPath = desired.SYNC_ARCHIVE && path.resolve(os.tmpdir(), desired.SYNC_ARCHIVE); - let archive, archiveRootFolder; - switch (options.platform) { - case 'mac': - serverFolder += `/macos/${flavor}`; - archive = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-Darwin-devel.tar.gz`; - break; - case 'ios': - flavor = flavor === 'Debug' ? 'MinSizeDebug' : flavor; - serverFolder += `/ios/${flavor}`; - archive = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-iphoneos.tar.gz`; - break; - case 'win': - if (!options.arch) throw new Error(`Specifying '--arch' is required for platform 'win'`); - const arch = options.arch === 'ia32' ? 'Win32' : options.arch; - serverFolder += `/windows/${arch}/nouwp/${flavor}`; - archive = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-Windows-${arch}-devel.tar.gz`; - break; - case 'linux': - serverFolder = 'core'; - archive = `realm-core-${dependencies.REALM_CORE_VERSION}.tgz`; - archiveRootFolder = `realm-core-${dependencies.REALM_CORE_VERSION}`; - break; - } - - const downloadedArchive = path.resolve(os.tmpdir(), archive); - return download(serverFolder, archive, downloadedArchive) - .then(() => extract(downloadedArchive, target, archiveRootFolder)); + return fs.emptyDir(target) + .then(() => corePath && download(desired.CORE_SERVER_FOLDER, desired.CORE_ARCHIVE, corePath)) + .then(() => corePath && extract(corePath, target, desired.CORE_ARCHIVE_ROOT)) + .then(() => syncPath && download(desired.SYNC_SERVER_FOLDER, desired.SYNC_ARCHIVE, syncPath)) + .then(() => syncPath && extract(syncPath, target, desired.SYNC_ARCHIVE_ROOT)) + .then(() => writeManifest(target, desired)) } function getSyncCommitSha(version) { @@ -156,36 +137,92 @@ function getSyncCommitSha(version) { }).then(stdout => /([^\t]+)/.exec(stdout)[0]); } -function acquireSync(options, dependencies, target) { - let serverFolder = 'sync'; +function appendCoreRequirements(desired, dependencies, options) { + desired.CORE_SERVER_FOLDER = `core/v${dependencies.REALM_CORE_VERSION}`; let flavor = options.debug ? 'Debug' : 'Release'; - let archive, archiveRootFolder; - let promise = Promise.resolve(); switch (options.platform) { - case 'mac': - archive = `realm-sync-node-cocoa-${dependencies.REALM_SYNC_VERSION}.tar.gz`; - archiveRootFolder = `realm-sync-node-cocoa-${dependencies.REALM_SYNC_VERSION}`; - break; - case 'ios': - archive = `realm-sync-cocoa-${dependencies.REALM_SYNC_VERSION}.tar.xz`; - archiveRootFolder = `core`; - break; - case 'win': - promise = acquireCore(options, dependencies, target) - .then(() => getSyncCommitSha(dependencies.REALM_SYNC_VERSION)) - .then(sha => serverFolder += `/sha-version/${sha}`); - const arch = options.arch === 'ia32' ? 'Win32' : options.arch; - archive = `realm-sync-${flavor}-v${dependencies.REALM_SYNC_VERSION}-Windows-${arch}-devel.tar.gz`; - break; - default: - throw new Error(`Unsupported sync platform '${options.platform}'`); + case 'mac': + desired.CORE_SERVER_FOLDER += `/macos/${flavor}`; + desired.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-Darwin-devel.tar.gz`; + return Promise.resolve(); + case 'ios': + flavor = flavor === 'Debug' ? 'MinSizeDebug' : flavor; + desired.CORE_SERVER_FOLDER += `/ios/${flavor}`; + desired.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-iphoneos.tar.gz`; + return Promise.resolve(); + case 'win': + if (!options.arch) throw new Error(`Specifying '--arch' is required for platform 'win'`); + const arch = options.arch === 'ia32' ? 'Win32' : options.arch; + desired.CORE_SERVER_FOLDER += `/windows/${arch}/nouwp/${flavor}`; + desired.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-Windows-${arch}-devel.tar.gz`; + return Promise.resolve(); + case 'linux': + desired.CORE_SERVER_FOLDER = 'core'; + desired.CORE_ARCHIVE = `realm-core-${dependencies.REALM_CORE_VERSION}.tgz`; + desired.CORE_ARCHIVE_ROOT = `realm-core-${dependencies.REALM_CORE_VERSION}`; + return Promise.resolve(); + default: + return Promise.reject(new Error(`Unsupported sync platform '${options.platform}'`)); } +} - const downloadedArchive = path.resolve(os.tmpdir(), archive); - return promise - .then(() => download(serverFolder, archive, downloadedArchive)) - .then(() => extract(downloadedArchive, target, archiveRootFolder)); +function appendSyncRequirements(desired, dependencies, options) { + desired.SYNC_SERVER_FOLDER = 'sync'; + let flavor = options.debug ? 'Debug' : 'Release'; + + switch (options.platform) { + case 'mac': + desired.SYNC_ARCHIVE = `realm-sync-node-cocoa-${dependencies.REALM_SYNC_VERSION}.tar.gz`; + desired.SYNC_ARCHIVE_ROOT = `realm-sync-node-cocoa-${dependencies.REALM_SYNC_VERSION}`; + return Promise.resolve(); + case 'ios': + desired.SYNC_ARCHIVE = `realm-sync-cocoa-${dependencies.REALM_SYNC_VERSION}.tar.xz`; + desired.SYNC_ARCHIVE_ROOT = `core`; + return Promise.resolve(); + case 'win': + const arch = options.arch === 'ia32' ? 'Win32' : options.arch; + desired.SYNC_ARCHIVE = `realm-sync-${flavor}-v${dependencies.REALM_SYNC_VERSION}-Windows-${arch}-devel.tar.gz`; + return appendCoreRequirements(desired, dependencies, options) + .then(() => getSyncCommitSha(dependencies.REALM_SYNC_VERSION)) + .then(sha => desired.SYNC_SERVER_FOLDER += `/sha-version/${sha}`); + default: + return Promise.reject(new Error(`Unsupported sync platform '${options.platform}'`)); + } +} + +function calculateRequirements(options, dependencies) { + const desired = {}; + return (options.sync ? appendSyncRequirements : appendCoreRequirements)(desired, dependencies, options) + .then(() => desired); +} + +function writeManifest(target, manifest) { + return fs.writeFile(path.resolve(target, MANIFEST_FILENAME), ini.encode(manifest)); +} + +function readManifest(target) { + try { + return ini.parse(fs.readFileSync(path.resolve(target, MANIFEST_FILENAME), 'utf8')); + } catch (e) { + return false; + } +} + +function shouldSkipAcquire(target, desiredManifest, force) { + const existingManifest = readManifest(target); + + if (!existingManifest) { + console.log('No manifest at the target, proceeding.'); + } else if (!Object.keys(desiredManifest).every(key => existingManifest[key] === desiredManifest[key])) { + console.log('Target has is non-empty but has a different manifest, overwriting.'); + } else if (force) { + console.log('Target has a matching manifest but download forced.'); + } else { + console.log('Matching manifest already exists at target - nothing to do (use --force to override)'); + return true; + } + return false; } const optionDefinitions = [ @@ -193,6 +230,7 @@ const optionDefinitions = [ { name: 'arch', type: String }, { name: 'sync', type: Boolean }, { name: 'debug', type: Boolean }, + { name: 'force', type: Boolean }, ]; const options = require('command-line-args')(optionDefinitions); if (options.platform === '..\\win') { @@ -209,9 +247,17 @@ if (options.debug) { realmDir += '-dbg' } -const dependencies = ini(fs.readFileSync(path.resolve(__dirname, '../dependencies.list'), 'utf8')); +const dependencies = ini.parse(fs.readFileSync(path.resolve(__dirname, '../dependencies.list'), 'utf8')); -const acquire = options.sync ? acquireSync : acquireCore; -fs.emptyDir(realmDir) - .then(() => acquire(options, dependencies, realmDir)) - .catch(error => console.error(error)); +calculateRequirements(options, dependencies) + .then(manifest => { + console.log('Desired manifest', manifest); + if (!shouldSkipAcquire(realmDir, manifest, options.force)) { + return acquire(manifest, realmDir) + } + }) + .then(() => console.log('Success')) + .catch(error => { + console.error(error); + process.exit(1); + }); From a5eed7297b4d39d665d66be99728cf19f3b8d1cb Mon Sep 17 00:00:00 2001 From: rh389 Date: Tue, 24 Oct 2017 13:05:42 +0100 Subject: [PATCH 2/3] Tidy up shouldSkipAcquire, skip manifest check on --force --- scripts/download-realm.js | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/scripts/download-realm.js b/scripts/download-realm.js index 0361ce86..b962af2c 100644 --- a/scripts/download-realm.js +++ b/scripts/download-realm.js @@ -205,24 +205,30 @@ function readManifest(target) { try { return ini.parse(fs.readFileSync(path.resolve(target, MANIFEST_FILENAME), 'utf8')); } catch (e) { - return false; + return null; } } function shouldSkipAcquire(target, desiredManifest, force) { + if (force) { + console.log('Skipping manifest check as --force is enabled'); + return false; + } + const existingManifest = readManifest(target); if (!existingManifest) { console.log('No manifest at the target, proceeding.'); - } else if (!Object.keys(desiredManifest).every(key => existingManifest[key] === desiredManifest[key])) { - console.log('Target has is non-empty but has a different manifest, overwriting.'); - } else if (force) { - console.log('Target has a matching manifest but download forced.'); - } else { - console.log('Matching manifest already exists at target - nothing to do (use --force to override)'); - return true; + return false; } - return false; + + if (!Object.keys(desiredManifest).every(key => existingManifest[key] === desiredManifest[key])) { + console.log('Target directory has a differing manifest, overwriting.'); + return false; + } + + console.log('Matching manifest already exists at target - nothing to do (use --force to override)'); + return true; } const optionDefinitions = [ From 345d81dad9c9d880d7969a5d15d8cc75829f4530 Mon Sep 17 00:00:00 2001 From: rh389 Date: Tue, 24 Oct 2017 13:29:04 +0100 Subject: [PATCH 3/3] Change vocabulary desired/manifest -> required/lockfile. Change appenders to getters. --- scripts/download-realm.js | 101 ++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 52 deletions(-) diff --git a/scripts/download-realm.js b/scripts/download-realm.js index b962af2c..a0d18677 100644 --- a/scripts/download-realm.js +++ b/scripts/download-realm.js @@ -26,7 +26,7 @@ const fetch = require('node-fetch'); const ini = require('ini'); const decompress = require('decompress'); -const MANIFEST_FILENAME = '.manifest.list'; +const LOCKFILE_NAME = 'download-realm.lock'; function exec() { const args = Array.from(arguments); @@ -123,7 +123,7 @@ function acquire(desired, target) { .then(() => corePath && extract(corePath, target, desired.CORE_ARCHIVE_ROOT)) .then(() => syncPath && download(desired.SYNC_SERVER_FOLDER, desired.SYNC_ARCHIVE, syncPath)) .then(() => syncPath && extract(syncPath, target, desired.SYNC_ARCHIVE_ROOT)) - .then(() => writeManifest(target, desired)) + .then(() => writeLockfile(target, desired)) } function getSyncCommitSha(version) { @@ -137,97 +137,94 @@ function getSyncCommitSha(version) { }).then(stdout => /([^\t]+)/.exec(stdout)[0]); } -function appendCoreRequirements(desired, dependencies, options) { - desired.CORE_SERVER_FOLDER = `core/v${dependencies.REALM_CORE_VERSION}`; +function getCoreRequirements(dependencies, options, required = {}) { + required.CORE_SERVER_FOLDER = `core/v${dependencies.REALM_CORE_VERSION}`; let flavor = options.debug ? 'Debug' : 'Release'; switch (options.platform) { case 'mac': - desired.CORE_SERVER_FOLDER += `/macos/${flavor}`; - desired.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-Darwin-devel.tar.gz`; - return Promise.resolve(); + required.CORE_SERVER_FOLDER += `/macos/${flavor}`; + required.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-Darwin-devel.tar.gz`; + return Promise.resolve(required); case 'ios': flavor = flavor === 'Debug' ? 'MinSizeDebug' : flavor; - desired.CORE_SERVER_FOLDER += `/ios/${flavor}`; - desired.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-iphoneos.tar.gz`; - return Promise.resolve(); + required.CORE_SERVER_FOLDER += `/ios/${flavor}`; + required.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-iphoneos.tar.gz`; + return Promise.resolve(required); case 'win': if (!options.arch) throw new Error(`Specifying '--arch' is required for platform 'win'`); const arch = options.arch === 'ia32' ? 'Win32' : options.arch; - desired.CORE_SERVER_FOLDER += `/windows/${arch}/nouwp/${flavor}`; - desired.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-Windows-${arch}-devel.tar.gz`; - return Promise.resolve(); + required.CORE_SERVER_FOLDER += `/windows/${arch}/nouwp/${flavor}`; + required.CORE_ARCHIVE = `realm-core-${flavor}-v${dependencies.REALM_CORE_VERSION}-Windows-${arch}-devel.tar.gz`; + return Promise.resolve(required); case 'linux': - desired.CORE_SERVER_FOLDER = 'core'; - desired.CORE_ARCHIVE = `realm-core-${dependencies.REALM_CORE_VERSION}.tgz`; - desired.CORE_ARCHIVE_ROOT = `realm-core-${dependencies.REALM_CORE_VERSION}`; - return Promise.resolve(); + required.CORE_SERVER_FOLDER = 'core'; + required.CORE_ARCHIVE = `realm-core-${dependencies.REALM_CORE_VERSION}.tgz`; + required.CORE_ARCHIVE_ROOT = `realm-core-${dependencies.REALM_CORE_VERSION}`; + return Promise.resolve(required); default: - return Promise.reject(new Error(`Unsupported sync platform '${options.platform}'`)); + return Promise.reject(new Error(`Unsupported core platform '${options.platform}'`)); } } -function appendSyncRequirements(desired, dependencies, options) { - desired.SYNC_SERVER_FOLDER = 'sync'; +function getSyncRequirements(dependencies, options, required = {}) { + required.SYNC_SERVER_FOLDER = 'sync'; let flavor = options.debug ? 'Debug' : 'Release'; switch (options.platform) { case 'mac': - desired.SYNC_ARCHIVE = `realm-sync-node-cocoa-${dependencies.REALM_SYNC_VERSION}.tar.gz`; - desired.SYNC_ARCHIVE_ROOT = `realm-sync-node-cocoa-${dependencies.REALM_SYNC_VERSION}`; - return Promise.resolve(); + required.SYNC_ARCHIVE = `realm-sync-node-cocoa-${dependencies.REALM_SYNC_VERSION}.tar.gz`; + required.SYNC_ARCHIVE_ROOT = `realm-sync-node-cocoa-${dependencies.REALM_SYNC_VERSION}`; + return Promise.resolve(required); case 'ios': - desired.SYNC_ARCHIVE = `realm-sync-cocoa-${dependencies.REALM_SYNC_VERSION}.tar.xz`; - desired.SYNC_ARCHIVE_ROOT = `core`; - return Promise.resolve(); + required.SYNC_ARCHIVE = `realm-sync-cocoa-${dependencies.REALM_SYNC_VERSION}.tar.xz`; + required.SYNC_ARCHIVE_ROOT = `core`; + return Promise.resolve(required); case 'win': const arch = options.arch === 'ia32' ? 'Win32' : options.arch; - desired.SYNC_ARCHIVE = `realm-sync-${flavor}-v${dependencies.REALM_SYNC_VERSION}-Windows-${arch}-devel.tar.gz`; - return appendCoreRequirements(desired, dependencies, options) + required.SYNC_ARCHIVE = `realm-sync-${flavor}-v${dependencies.REALM_SYNC_VERSION}-Windows-${arch}-devel.tar.gz`; + return getCoreRequirements(dependencies, options, required) .then(() => getSyncCommitSha(dependencies.REALM_SYNC_VERSION)) - .then(sha => desired.SYNC_SERVER_FOLDER += `/sha-version/${sha}`); + .then(sha => { + required.SYNC_SERVER_FOLDER += `/sha-version/${sha}`; + return required; + }); default: return Promise.reject(new Error(`Unsupported sync platform '${options.platform}'`)); } } -function calculateRequirements(options, dependencies) { - const desired = {}; - return (options.sync ? appendSyncRequirements : appendCoreRequirements)(desired, dependencies, options) - .then(() => desired); +function writeLockfile(target, contents) { + return fs.writeFile(path.resolve(target, LOCKFILE_NAME), ini.encode(contents)); } -function writeManifest(target, manifest) { - return fs.writeFile(path.resolve(target, MANIFEST_FILENAME), ini.encode(manifest)); -} - -function readManifest(target) { +function readLockfile(target) { try { - return ini.parse(fs.readFileSync(path.resolve(target, MANIFEST_FILENAME), 'utf8')); + return ini.parse(fs.readFileSync(path.resolve(target, LOCKFILE_NAME), 'utf8')); } catch (e) { return null; } } -function shouldSkipAcquire(target, desiredManifest, force) { +function shouldSkipAcquire(target, requirements, force) { if (force) { - console.log('Skipping manifest check as --force is enabled'); + console.log('Skipping lockfile check as --force is enabled'); return false; } - const existingManifest = readManifest(target); + const existingLockfile = readLockfile(target); - if (!existingManifest) { - console.log('No manifest at the target, proceeding.'); + if (!existingLockfile) { + console.log('No lockfile found at the target, proceeding.'); return false; } - if (!Object.keys(desiredManifest).every(key => existingManifest[key] === desiredManifest[key])) { - console.log('Target directory has a differing manifest, overwriting.'); + if (!Object.keys(requirements).every(key => existingLockfile[key] === requirements[key])) { + console.log('Target directory has a differing lockfile, overwriting.'); return false; } - console.log('Matching manifest already exists at target - nothing to do (use --force to override)'); + console.log('Matching lockfile already exists at target - nothing to do (use --force to override)'); return true; } @@ -255,11 +252,11 @@ if (options.debug) { const dependencies = ini.parse(fs.readFileSync(path.resolve(__dirname, '../dependencies.list'), 'utf8')); -calculateRequirements(options, dependencies) - .then(manifest => { - console.log('Desired manifest', manifest); - if (!shouldSkipAcquire(realmDir, manifest, options.force)) { - return acquire(manifest, realmDir) +(options.sync ? getSyncRequirements : getCoreRequirements)(dependencies, options) + .then(requirements => { + console.log('Resolved requirements:', requirements); + if (!shouldSkipAcquire(realmDir, requirements, options.force)) { + return acquire(requirements, realmDir) } }) .then(() => console.log('Success'))