feat: add support for embark.config.js

This commit introduces support for using `embark.config.js` to calculate the
embark configuration object that is otherwise provided via `embark.json`.

If an `embark.config.js` file is present, it will be used over the
`embark.json` file.  The `embark.config.js` module needs to export either an
object or a function that can be asynchronous and has to return or resolve with
an embark configuration object:

```js
// embark.config.js

module.exports = async function () {
  let config = ...; // do lazy calculation of `embarkConfig`;
  return config;
}
```

packages/core/console/src/index.ts

@@ -169,7 +169,7 @@ export default class Console {
       return this.ipc.request("console:executeCmd", cmd, callback);
     }
 
-    if (cmd.indexOf("profile") === 0 && warnIfPackageNotDefinedLocally("embark-profiler", this.embark.logger.warn) !== true) {
+    if (cmd.indexOf("profile") === 0 && warnIfPackageNotDefinedLocally("embark-profiler", this.embark.logger.warn, this.embark.config.embarkConfig) !== true) {
       return callback(null, "please install embark-profiler plugin");
     }
     if (!(cmd.split(" ")[0] === "history" || cmd === __("history"))) {

packages/core/engine/src/index.ts

@@ -121,8 +121,8 @@ export class Engine {
   }
 
   async generateEmbarkJSON() {
-    if (fs.existsSync('embark.json')) {
-      throw new Error(__('embark.json already there. Will not overwrite'));
+    if (fs.existsSync('embark.json') || fs.existsSync('embark.config.js')) {
+      throw new Error(__('embark.json or embark.config.js already exist. Will not overwrite'));
     }
     return fs.writeFile('embark.json', JSON.stringify(constants.defaultEmbarkConfig, null, 2));
   }

packages/core/utils/src/index.ts

@@ -313,7 +313,7 @@ export function isEs6Module(module) {
   return (typeof module === 'function' && isConstructor(module)) || (typeof module === 'object' && typeof module.default === 'function' && module.__esModule);
 }
 
-export function warnIfPackageNotDefinedLocally(packageName, warnFunc) {
+export function warnIfPackageNotDefinedLocally(packageName, warnFunc, embarkConfig) {
   const packageIsResolvable = findUp.sync("node_modules/" + packageName, {cwd: dappPath()});
   if (!packageIsResolvable) {
     return warnFunc("== WARNING: "  + packageName + " could not be resolved; ensure it is defined in your dapp's package.json dependencies and then run npm or yarn install; in future versions of embark this package should be a local dependency and configured as a plugin");
@@ -325,7 +325,6 @@ export function warnIfPackageNotDefinedLocally(packageName, warnFunc) {
     return warnFunc("== WARNING: it seems "  + packageName + " is not defined in your dapp's package.json dependencies; In future versions of embark this package should be a local dependency and configured as a plugin");
   }
 
-  const embarkConfig = fs.readJSONSync(dappPath("embark.json"));
   if (!embarkConfig.plugins[packageName]) {
     return warnFunc(
       __("== WARNING: it seems %s is not defined in your Dapp's embark.json plugins;\nIn future versions of Embark, this package should be a local dependency and configured as a plugin", packageName)

packages/embark/src/bin/embark.js

@@ -4,11 +4,18 @@
 // on babel to achieve the same goal.
 // See: https://node.green/
 
+// KEY ASSUMPTION: for a DApp to be valid, from embark's cli's perspective, it
+// must have an importable embark.config.js file or a parsable embark.json file
+// in its top-level directory; if that requirement changes in the future then
+// this script must be revised. Hypothetical example of such a change: embark
+// config info may be included in package.json under `{"embark": {...}}` -or-
+// stored in a config file.
+
 function main() {
   if (whenNoShim()) return;
   var invoked = thisEmbark();
-  var embarkJson = findEmbarkJson();
-  var dappPath = embarkJson.dirname;
+  var embarkConfig = findEmbarkConfig();
+  var dappPath = embarkConfig.dirname;
   process.chdir(dappPath);
   process.env.DAPP_PATH = dappPath;
   process.env.PWD = dappPath;
@@ -36,11 +43,11 @@ function main() {
   var containing = findBinContaining(dappPath, invoked);
   var installed = findBinInstalled(dappPath, invoked);
   var local = selectLocal(containing, installed, invoked);
-  var pkgJson = findPkgJson(dappPath, embarkJson, local);
+  var pkgJson = findPkgJson(dappPath, embarkConfig, local);
   process.env.PKG_PATH = pkgJson.dirname;
   var embark = select(invoked, local);
   process.env.EMBARK_PATH = embark.pkgDir;
-  embark.exec(embarkJson);
+  embark.exec(embarkConfig);
 }
 
 // -----------------------------------------------------------------------------
@@ -71,11 +78,31 @@ function EmbarkBin(binpath, kind) {
   this.pkgJson = undefined;
 }
 
-EmbarkBin.prototype.exec = function (embarkJson) {
+EmbarkBin.prototype.exec = function (embarkConfig) {
+  if (!embarkConfig) {
+    embarkConfig = findEmbarkConfig();
+    var dappPath = embarkConfig.dirname;
+    process.chdir(dappPath);
+    process.env.DAPP_PATH = dappPath;
+    process.env.PWD = dappPath;
+  }
   var Cmd = require('../cmd/cmd');
-  var cli = new Cmd({embarkConfig: embarkJson.json});
-  if(_logged) { console[this.loglevel](); }
+  var embarkBin = this;
+  Promise.resolve(embarkConfig.contents)
+    .then(function (embarkConfig) {
+      try {
+        var cli = new Cmd({embarkConfig});
+        if(_logged) { console[embarkBin.loglevel](); }
         cli.process(process.argv);
+      } catch (e) {
+        reportUnhandledCliException(e);
+        exitWithError();
+      }
+    })
+    .catch(function (e) {
+      reportConfigRejected(embarkConfig.filepath, e);
+      exitWithError();
+    });
 };
 
 EmbarkBin.prototype.handle = function () {
@@ -266,21 +293,39 @@ function findBinInstalled(dappPath, invoked) {
   );
 }
 
-function findEmbarkJson() {
-  // findUp search begins in process.cwd() by default, but embark.json could
-  // be in a subdir if embark was invoked via `npm run` (which changes cwd to
+function findEmbarkConfig() {
+  // findUp search begins in process.cwd() by default, but the config could be
+  // in a subdir if embark was invoked via `npm run` (which changes cwd to
   // package.json's dir) and the package.json is in a dir above the top-level
   // DApp dir; so start at INIT_CWD if that has been set (by npm, presumably)
   // See: https://docs.npmjs.com/cli/run-script
+  var cmd = process.argv[2];
+  var embarkConfig, embarkConfigJsPath, embarkJsonPath;
   var startDir = initCwd();
-  return (new EmbarkJson(
-    findUp.sync('embark.json', {cwd: startDir}) ||
-      path.join(startDir, 'embark.json'),
-    process.argv[2]
-  )).handle();
+  embarkConfigJsPath = findUp.sync('embark.config.js', {cwd: startDir});
+  if (embarkConfigJsPath) {
+    embarkConfig = new EmbarkConfig(
+      (new EmbarkConfigJs(embarkConfigJsPath, cmd)).handle()
+    );
+  } else {
+    embarkJsonPath = findUp.sync('embark.json', {cwd: startDir});
+    if (embarkJsonPath) {
+      embarkConfig = new EmbarkConfig(
+        (new EmbarkJson(embarkJsonPath, cmd)).handle()
+      );
+    }
+  }
+  if (!embarkConfig) {
+    embarkConfig = new EmbarkConfig(
+      (new EmbarkConfigJs(
+        path.join(startDir, 'embark.config.js'), cmd)
+      ).handle()
+    );
+  }
+  return embarkConfig.handle();
 }
 
-function findPkgJson(dappPath, embarkJson, local) {
+function findPkgJson(dappPath, embarkConfig, local) {
   var skipDirs = [];
   if (local) {
     if (local instanceof EmbarkBinLocalContaining) {
@@ -302,7 +347,7 @@ function findPkgJson(dappPath, embarkJson, local) {
       closest = found;
     }
     dir = found ? path.dirname(found) : found;
-    var stop = !dir || !isDappCmd(embarkJson.cmd);
+    var stop = !dir || !isDappCmd(embarkConfig.cmd);
     if (!stop) {
       startDir = path.join(dir, '..');
     }
@@ -313,10 +358,10 @@ function findPkgJson(dappPath, embarkJson, local) {
       (new PkgJsonLocal(found)).handle();
     }
   }
-  if (isDappCmd(embarkJson.cmd) && !closest) {
+  if (isDappCmd(embarkConfig.cmd) && !closest) {
     var loglevel = 'error';
     reportMissingFile(path.join(dappPath, 'package.json'), loglevel);
-    reportMissingFile_DappJson(embarkJson.cmd, loglevel, 'package', 'in or above');
+    reportMissingFile_DappJson(embarkConfig.cmd, loglevel, 'package', 'in or above');
     exitWithError();
   }
   return (
@@ -324,6 +369,128 @@ function findPkgJson(dappPath, embarkJson, local) {
   );
 }
 
+// -- generic config -----------------------------------------------------------
+
+function EmbarkConfig(configFile) {
+  this.configFile = configFile;
+  this.filepath = configFile.filepath;
+  this.cmd = configFile.cmd;
+  this.dirname = configFile.dirname;
+  this.realpath = configFile.realpath;
+  this.callError = undefined;
+  this.contents = undefined;
+}
+
+EmbarkConfig.prototype.handle = function () {
+  this.setup();
+  this.log();
+  return this;
+};
+
+EmbarkConfig.prototype.log = function () {
+  this.logMissingConfig();
+  this.logCallFailed();
+};
+
+EmbarkConfig.prototype.loglevel = 'error';
+
+EmbarkConfig.prototype.logCallFailed = function () {
+  if (isDappCmd(this.cmd) && this.realpath && this.callError) {
+    reportCallFailed(this.filepath, this.callError, this.loglevel);
+    exitWithError();
+  }
+};
+
+EmbarkConfig.prototype.logMissingConfig = function () {
+  if (isDappCmd(this.cmd) && !this.realpath) {
+    reportMissingConfig(this.cmd, this.loglevel);
+    exitWithError();
+  }
+};
+
+EmbarkConfig.prototype.setContents = function () {
+  if (isDappCmd(this.cmd) && this.realpath) {
+    if (this.configFile instanceof EmbarkConfigJs) {
+      this.contents = this.configFile.exports;
+      if (typeof this.contents === 'function') {
+        try {
+          this.contents = this.contents();
+        } catch (e) {
+          this.contents = undefined;
+          this.callError = e;
+        }
+      }
+    } else if (this.configFile instanceof EmbarkJson) {
+      this.contents = this.configFile.json;
+    }
+  }
+};
+
+EmbarkConfig.prototype.setup = function () {
+  this.setContents();
+  return this;
+};
+
+// -- js config file -----------------------------------------------------------
+
+function EmbarkConfigJs(filepath, cmd) {
+  this.filepath = filepath;
+  this.cmd = cmd;
+  this.dirname = undefined;
+  this.exports = undefined;
+  this.importError = undefined;
+  this.realpath = undefined;
+}
+
+EmbarkConfigJs.prototype.handle = function () {
+  this.setup();
+  this.log();
+  return this;
+};
+
+EmbarkConfigJs.prototype.log = function () {
+  this.logUnimportable();
+};
+
+EmbarkConfigJs.prototype.loglevel = 'error';
+
+EmbarkConfigJs.prototype.logUnimportable = function () {
+  if (isDappCmd(this.cmd) && this.realpath && this.importError) {
+    reportUnimportable(this.filepath, this.importError, this.loglevel);
+    exitWithError();
+  }
+};
+
+EmbarkConfigJs.prototype.setDirname = function () {
+  if (this.filepath) {
+    this.dirname = path.dirname(this.filepath);
+  }
+};
+
+EmbarkConfigJs.prototype.setExports = function () {
+  if (isDappCmd(this.cmd) && this.realpath) {
+    try {
+      this.exports = require(this.filepath);
+    } catch (e) {
+      this.exports = undefined;
+      this.importError = e;
+    }
+  }
+};
+
+EmbarkConfigJs.prototype.setRealpath = function () {
+  if (this.filepath) {
+    this.realpath = realpath(this.filepath);
+  }
+};
+
+EmbarkConfigJs.prototype.setup = function () {
+  this.setDirname();
+  this.setRealpath();
+  this.setExports();
+  return this;
+};
+
 // -- json files ---------------------------------------------------------------
 
 function Json(filepath) {
@@ -346,17 +513,12 @@ Json.prototype.log = function () {
 
 Json.prototype.loglevel = 'warn';
 
-Json.prototype.logMissingFile = function (doReport) {
-  if (doReport === undefined) {
-    doReport = true;
-  }
+Json.prototype.logMissingFile = function () {
   var missing;
   if (!this.realpath) {
     missing = true;
-    if (doReport) {
     reportMissingFile(this.filepath, this.loglevel);
   }
-  }
   return missing;
 };
 
@@ -405,20 +567,9 @@ setupProto(EmbarkJson, Json);
 EmbarkJson.prototype.loglevel = 'error';
 
 EmbarkJson.prototype.log = function () {
-  this.logMissingFile();
   this.logUnparsable();
 };
 
-EmbarkJson.prototype.logMissingFile = function () {
-  if (Json.prototype.logMissingFile.call(this, false)) {
-    if (isDappCmd(this.cmd)) {
-      embarklog['error']('No embark.json file found.\n' +
-        'Run `embark init` to generate one automatically.');
-      exitWithError();
-    }
-  }
-};
-
 EmbarkJson.prototype.logUnparsable = function () {
   if (isDappCmd(this.cmd) && Json.prototype.logUnparsable.call(this)) {
     reportUnparsable_EmbarkJson(this.loglevel);
@@ -737,6 +888,43 @@ function whenNoShim() {
 
 // -- reporters ----------------------------------------------------------------
 
+function reportCallFailed(filepath, error, loglevel) {
+  var basename = path.basename(filepath);
+  blankLineMaybe(loglevel);
+  embarklog[loglevel]('file', filepath);
+  if (error.code) embarklog[loglevel]('code', error.code);
+  embarklog[loglevel](
+    'exports',
+    `An exception was thrown when calling the function exported by ${basename}`
+  );
+  embarklog[loglevel]('', error.stack);
+}
+
+function reportConfigRejected(filepath, error, loglevel) {
+  if (!loglevel) loglevel = 'error';
+  var basename = path.basename(filepath);
+  blankLineMaybe(loglevel);
+  embarklog[loglevel]('file', filepath);
+  if (error.code) embarklog[loglevel]('code', error.code);
+  embarklog[loglevel](
+    'exports',
+    `Promise rejection when resolving the configuration from ${basename}`
+  );
+  embarklog[loglevel]('', error.stack);
+}
+
+function reportMissingConfig(cmd, loglevel) {
+  blankLineMaybe(loglevel);
+  embarklog[loglevel](
+    '',
+    `Could not locate your DApp's embark.config.js or embark.json file`
+  );
+  embarklog[loglevel](
+    '',
+    'Run `embark init` to generate an embark.json file automatically'
+  );
+}
+
 function reportMissingEmbarkDep(filepath, dirname, loglevel) {
   blankLineMaybe(loglevel);
   embarklog[loglevel]('file', filepath);
@@ -870,6 +1058,21 @@ function reportSwitching(binpathFrom, binpathTo, loglevel, pkgFrom, pkgTo) {
   );
 }
 
+function reportUnhandledCliException(error, loglevel) {
+  if (!loglevel) loglevel = 'error';
+  blankLineMaybe(loglevel);
+  embarklog[loglevel]('', 'Unhandled exception in the command-line interface');
+  embarklog[loglevel]('', error.stack);
+}
+
+function reportUnimportable(filepath, error, loglevel) {
+  blankLineMaybe(loglevel);
+  embarklog[loglevel]('file', filepath);
+  if (error.code) embarklog[loglevel]('code', error.code);
+  embarklog[loglevel]('require', `Failed to import module`);
+  embarklog[loglevel]('', error.stack);
+}
+
 function reportUnparsable(filepath, loglevel) {
   try {
     // force the exception

packages/plugins/basic-pipeline/src/index.js

@@ -117,7 +117,8 @@ class BasicPipeline {
           options: {
             webpackConfigName: self.webpackConfigName,
             pipelineConfig: self.pipelineConfig,
-            fs: self.embark.fs
+            fs: self.embark.fs,
+            embarkConfig: self.embark.config.embarkConfig
           }
         });
         webpackProcess.send({action: constants.pipeline.build, assets: self.assetFiles, importsList});

packages/plugins/basic-pipeline/src/webpack.config.js

@@ -54,9 +54,9 @@ const embarkAliases = require(path.join(dappPath, '.embark/embark-aliases.json')
 const embarkAssets = require(path.join(dappPath, '.embark/embark-assets.json'));
 let embarkJson;
 try {
-  embarkJson = require(path.join(dappPath, 'embark.json'));
-}catch (e) {
-  throw new Error('embark.json not found. To build your app, please add an embark.json file with a `buildDir` field');
+  embarkJson = require(path.join(dappPath, '.embark', 'embark.json'));
+} catch (e) {
+  throw new Error(`embark.json not found in ${path.join(dappPath, '.embark')}/`);
 }
 
 const embarkPipeline = require(path.join(dappPath, '.embark/embark-pipeline.json'));

packages/plugins/basic-pipeline/src/webpackProcess.js

@@ -13,6 +13,7 @@ class WebpackProcess extends ProcessWrapper {
     super(options);
     this.webpackConfigName = options.webpackConfigName;
     this.pipelineConfig = options.pipelineConfig;
+    this.embarkConfig = options.embarkConfig;
   }
 
   async build(assets, importsList, callback) {
@@ -25,6 +26,10 @@ class WebpackProcess extends ProcessWrapper {
 
   async webpackRun(assets, importsList, callback) {
     try {
+      await writeFile(
+        dappPath('.embark/embark.json'),
+        JSON.stringify(this.embarkConfig)
+      );
       await writeFile(
         dappPath('.embark/embark-aliases.json'),
         JSON.stringify(importsList)

packages/plugins/graph/src/index.js

@@ -9,7 +9,7 @@ class GraphGenerator {
     this.events = embark.events;
 
     this.events.setCommandHandler("graph:create", this.generate.bind(this));
-    warnIfPackageNotDefinedLocally("embark-graph", this.embark.logger.warn.bind(this.embark.logger));
+    warnIfPackageNotDefinedLocally("embark-graph", this.embark.logger.warn.bind(this.embark.logger), this.embark.config.embarkConfig);
   }
 
   /*eslint complexity: ["error", 21]*/

packages/stack/blockchain/src/index.js

@@ -161,10 +161,10 @@ export default class Blockchain {
     this.blockchainApi.registerRequests("ethereum");
 
     if (this.blockchainConfig.enabled && this.blockchainConfig.client === "geth") {
-      warnIfPackageNotDefinedLocally("embark-geth", this.embark.logger.warn.bind(this.embark.logger));
+      warnIfPackageNotDefinedLocally("embark-geth", this.embark.logger.warn.bind(this.embark.logger), this.embark.config.embarkConfig);
     }
     if (this.blockchainConfig.enabled && this.blockchainConfig.client === "parity") {
-      warnIfPackageNotDefinedLocally("embark-parity", this.embark.logger.warn.bind(this.embark.logger));
+      warnIfPackageNotDefinedLocally("embark-parity", this.embark.logger.warn.bind(this.embark.logger), this.embark.config.embarkConfig);
     }
   }
 

packages/stack/communication/src/index.js

@@ -32,7 +32,7 @@ class Communication {
     });
 
     if (this.communicationConfig.enabled && this.communicationConfig.available_providers.includes("whisper")) {
-      warnIfPackageNotDefinedLocally("embark-whisper-geth", this.embark.logger.warn.bind(this.embark.logger));
+      warnIfPackageNotDefinedLocally("embark-whisper-geth", this.embark.logger.warn.bind(this.embark.logger), this.embark.config.embarkConfig);
     }
   }
 

packages/stack/storage/src/index.js

@@ -44,10 +44,10 @@ class Storage {
     });
 
     if (this.storageConfig.enabled && this.storageConfig.available_providers.includes("ipfs")) {
-      warnIfPackageNotDefinedLocally("embark-ipfs", this.embark.logger.warn.bind(this.embark.logger));
+      warnIfPackageNotDefinedLocally("embark-ipfs", this.embark.logger.warn.bind(this.embark.logger), this.embark.config.embarkConfig);
     }
     if (this.storageConfig.enabled && this.storageConfig.available_providers.includes("swarm")) {
-      warnIfPackageNotDefinedLocally("embark-swarm", this.embark.logger.warn.bind(this.embark.logger));
+      warnIfPackageNotDefinedLocally("embark-swarm", this.embark.logger.warn.bind(this.embark.logger), this.embark.config.embarkConfig);
     }
   }
 

site/source/docs/configuration.md

@@ -35,7 +35,22 @@ Every application [created with Embark](create_project.html) comes with an `emba
 }
 ```
 
-Let's look at the different options and learn what they do and mean.
+Alternatively, it's possible to use a `embark.config.js` file, which exports either a configuration object or a function that calculates the object. This is particularly useful when the configuration needs to be built based on asynchronous operations. To make use of `embark.config.js`, simply create the file in your DApp and make sure it exports an (async) function which resolves with a config object like this:
+
+```js
+// embark.config.js
+
+module.exports = async function () {
+  const secrets = await getSecrets();
+  return {
+    // Embark configuration goes here
+  };
+};
+```
+
+Embark will import and run the exported function. If `module.exports` is a `Promise` or a function that returns a `Promise` (as above), Embark will automatically resolve the promised config object.
+
+Let's look at the different configuration options and learn what they do and mean.
 
 ### contracts
 

site/source/docs/structure.md

@@ -74,6 +74,10 @@ This file is used to keep track of the deployed contracts in each chain. This is
 
 In order to provide as much flexibility for our users as possible, Embark comes with an `embark.json` file that lets us configure our own directory structure. This file is also used to specify Embark plugins and other configurations. More information on how to use this configuration file, can be found in [configuring embark.json](configuration.html).
 
+### embark.config.js
+
+If we need more control over how our `embark.json` configuration object should be composed, `embark.config.js` can be used as a module that exports either a configuration object or a function that returns a configuration object. That way, we can perform any calculations needed. For more information, checkout the [embark.config.js section](configuration.html#embark.config.js).
+
 ## Smart Contracts only template structure
 
 When creating a project using the `--contracts-only` option, the resulting structure is a little bit simpler. Let's take quick look: