refactor(@embark/pipeline): move pipeline into own package

packages/embark-pipeline/.npmrc

@@ -0,0 +1,4 @@
+engine-strict = true
+package-lock = false
+save-exact = true
+scripts-prepend-node-path = true

packages/embark-pipeline/README.md

@@ -0,0 +1,6 @@
+# `embark-pipeline`
+
+> Build pipeline module for Embark
+
+Visit [embark.status.im](https://embark.status.im/) to get started with
+[Embark](https://github.com/embark-framework/embark).

packages/embark-pipeline/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "embark-pipeline",
+  "version": "4.1.0-beta.0",
+  "author": "Iuri Matias <iuri.matias@gmail.com>",
+  "contributors": [],
+  "description": "Build pipeline module for Embark",
+  "homepage": "https://github.com/embark-framework/embark/tree/master/packages/embark-pipeline#readme",
+  "bugs": "https://github.com/embark-framework/embark/issues",
+  "keywords": [
+    "blockchain",
+    "dapps",
+    "ethereum",
+    "ipfs",
+    "serverless",
+    "solc",
+    "solidity"
+  ],
+  "files": [
+    "dist"
+  ],
+  "license": "MIT",
+  "repository": {
+    "directory": "packages/embark-pipeline",
+    "type": "git",
+    "url": "https://github.com/embark-framework/embark.git"
+  },
+  "main": "./dist/index.js",
+  "scripts": {
+    "build": "cross-env BABEL_ENV=node babel src --copy-files --extensions \".js\" --out-dir dist --root-mode upward --source-maps",
+    "ci": "npm run qa",
+    "clean": "npm run reset",
+    "lint": "npm-run-all lint:*",
+    "lint:js": "eslint src/",
+    "// lint:ts": "tslint -c tslint.json \"src/**/*.ts\"",
+    "package": "npm pack",
+    "// qa": "npm-run-all lint typecheck build package",
+    "qa": "npm-run-all lint build package",
+    "reset": "npx rimraf dist embark-*.tgz package",
+    "start": "npm run watch",
+    "// typecheck": "tsc",
+    "watch": "run-p watch:*",
+    "watch:build": "npm run build -- --verbose --watch",
+    "// watch:typecheck": "npm run typecheck -- --preserveWatchOutput --watch"
+  },
+  "eslintConfig": {
+    "extends": "../../.eslintrc.json"
+  },
+  "dependencies": {
+    "@babel/runtime-corejs2": "7.3.1",
+    "async": "2.6.1",
+    "embark-core": "^4.1.0-beta.0",
+    "embark-i18n": "^4.1.0-beta.0",
+    "embark-utils": "^4.1.0-beta.0",
+    "find-up": "2.1.0",
+    "webpack": "4.29.3"
+  },
+  "devDependencies": {
+    "@babel/cli": "7.2.3",
+    "@babel/core": "7.2.2",
+    "cross-env": "5.2.0",
+    "eslint": "5.7.0",
+    "npm-run-all": "4.1.5",
+    "rimraf": "2.6.3",
+    "tslint": "5.16.0",
+    "typescript": "3.4.5"
+  },
+  "engines": {
+    "node": ">=8.12.0",
+    "npm": ">=6.4.1",
+    "yarn": ">=1.12.3"
+  }
+}

packages/embark-pipeline/src/index.js

@@ -1,11 +1,10 @@
 const path = require('path');
 const async = require('async');
-const utils = require('../../utils/utils.js');
 import { __ } from 'embark-i18n';
-import {joinPath, LongRunningProcessTimer} from 'embark-utils';
+import { joinPath, LongRunningProcessTimer, fileTreeSort } from 'embark-utils';
 import { dappPath, ProcessLauncher } from 'embark-core';
 const constants = require('embark-core/constants');
-const WebpackConfigReader = require('../pipeline/webpackConfigReader');
+const WebpackConfigReader = require('./webpackConfigReader');
 
 class Pipeline {
   constructor(embark, options) {
@@ -118,7 +117,7 @@ class Pipeline {
               dirname: dir,
               path: path.join(dir, name),
               isHidden: (name.indexOf('.') === 0 || name === "node_modules"),
-              children: utils.fileTreeSort(walk(path.join(dir, name), filelist))
+              children: fileTreeSort(walk(path.join(dir, name), filelist))
             };
           }
           return {
@@ -129,7 +128,7 @@ class Pipeline {
             isHidden: (name.indexOf('.') === 0 || name === "node_modules")
           };
         });
-        const files = utils.fileTreeSort(walk(dappPath()));
+        const files = fileTreeSort(walk(dappPath()));
         res.send(files);
       }
     );
@@ -237,7 +236,8 @@ class Pipeline {
           action: constants.pipeline.init,
           options: {
             webpackConfigName: self.webpackConfigName,
-            pipelineConfig: self.pipelineConfig
+            pipelineConfig: self.pipelineConfig,
+            fs: self.embark.fs
           }
         });
         webpackProcess.send({action: constants.pipeline.build, assets: self.assetFiles, importsList});

packages/embark-pipeline/src/webpackConfigReader.js

@@ -1,6 +1,7 @@
-import { dappPath, embarkPath } from 'embark-core';
-const {errorMessage} = require('../../utils/utils');
+import { dappPath } from 'embark-core';
+import { errorMessage } from 'embark-utils';
 const fs = require('fs-extra');
+import * as path from 'path';
 
 class WebpackConfigReader {
   constructor(options) {
@@ -9,7 +10,7 @@ class WebpackConfigReader {
 
   async readConfig(callback){
     const dappConfigPath = dappPath('webpack.config.js');
-    const defaultConfigPath = embarkPath('dist/lib/modules/pipeline', 'webpack.config.js');
+    const defaultConfigPath = path.resolve(__dirname, 'webpack.config.js');
 
     let config, configPath;
     try {

packages/embark-pipeline/src/webpackProcess.js

@@ -1,8 +1,8 @@
 import { dappPath, ProcessWrapper } from 'embark-core';
+import { errorMessage } from 'embark-utils';
 const constants = require('embark-core/constants');
 const webpack = require('webpack');
 const writeFile = require('util').promisify(require('fs').writeFile);
-const {errorMessage} = require('../../utils/utils');
 const WebpackConfigReader = require('./webpackConfigReader');
 
 let webpackProcess;

packages/embark-pipeline/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["src/**/*"]
+}

packages/embark-pipeline/tslint.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../../tslint.json"
+}

packages/embark-utils/src/index.js

@@ -109,6 +109,25 @@ function copyToClipboard(text) {
   clipboardy.writeSync(text);
 }
 
+function byName(a, b) {
+  return a.name.localeCompare(b.name);
+}
+
+function isFolder(node) {
+  return node.children && node.children.length;
+}
+
+function isNotFolder(node){
+  return !isFolder(node);
+}
+
+function fileTreeSort(nodes){
+  const folders = nodes.filter(isFolder).sort(byName);
+  const files = nodes.filter(isNotFolder).sort(byName);
+
+  return folders.concat(files);
+}
+
 function proposeAlternative(word, _dictionary, _exceptions) {
   const propose = require('propose');
   let exceptions = _exceptions || [];
@@ -226,6 +245,15 @@ function joinPath() {
 
 function tmpDir(...args) { return joinPath(os.tmpdir(), ...args); }
 
+function errorMessage(e) {
+  if (typeof e === 'string') {
+    return e;
+  } else if (e && e.message) {
+    return e.message;
+  }
+  return e;
+}
+
 const Utils = {
   buildUrl,
   buildUrlFromConfig,
@@ -242,11 +270,13 @@ const Utils = {
   decodeParams,
   dockerHostSwap,
   exit,
+  errorMessage,
   getAddressToContract,
   getTransactionParams,
   isDocker,
   checkIsAvailable,
   findNextPort,
+  fileTreeSort,
   hashTo32ByteHexString,
   hexToNumber,
   isHex,

packages/embark/package.json

@@ -108,6 +108,7 @@
     "embark-i18n": "^4.1.0-beta.0",
     "embark-library-manager": "^4.1.0-beta.0",
     "embark-logger": "^4.1.0-beta.0",
+    "embark-pipeline": "^4.1.0-beta.0",
     "embark-plugin-cmd": "^4.1.0-beta.0",
     "embark-process-logs-api": "^4.1.0-beta.0",
     "embark-profiler": "^4.1.0-beta.0",

packages/embark/src/cmd/cmd_controller.js

@@ -1,6 +1,7 @@
 import { BlockchainClient, Simulator } from 'embark-blockchain-process';
 import { dappPath, embarkPath } from 'embark-core';
 import { __ } from 'embark-i18n';
+import findUp from 'find-up';
 let async = require('async');
 const constants = require('embark-core/constants');
 const Logger = require('embark-logger');
@@ -450,13 +451,13 @@ class EmbarkController {
   }
 
   ejectWebpack() {
-    var embarkConfig = embarkPath('dist/lib/modules/pipeline/webpack.config.js');
-    var dappConfig = dappPath('webpack.config.js');
+    const embarkConfig = (findUp.sync('node_modules/embark-pipeline/dist/webpack.config.js', {cwd: embarkPath()}) || embarkPath("node_modules/embark-pipeline/dist/webpack.config.js"));
+    const dappConfig = dappPath('webpack.config.js');
     fs.copyPreserve(embarkConfig, dappConfig);
     console.log(__('webpack config ejected to:').dim.yellow);
     console.log(`${dappConfig}`.green);
-    var embarkOverrides = embarkPath('dist/lib/modules/pipeline/babel-loader-overrides.js');
-    var dappOverrides = dappPath('babel-loader-overrides.js');
+    const embarkOverrides = (findUp.sync('node_modules/embark-pipeline/dist/babel-loader-overrides.js', {cwd: embarkPath()}) || embarkPath("node_modules/embark-pipeline/dist/babel-loader-overrides.js"));
+    const dappOverrides = dappPath('babel-loader-overrides.js');
     fs.copyPreserve(embarkOverrides, dappOverrides);
     console.log(__('webpack overrides ejected to:').dim.yellow);
     console.log(`${dappOverrides}`.green);

packages/embark/src/lib/core/engine.js

@@ -142,7 +142,7 @@ class Engine {
 
   pipelineService(_options) {
     const self = this;
-    this.registerModule('pipeline', {
+    this.registerModulePackage('embark-pipeline', {
       webpackConfigName: this.webpackConfigName,
       useDashboard: this.useDashboard
     });

packages/embark/src/lib/utils/template_generator.js

@@ -4,7 +4,7 @@ const findUp = require('find-up');
 const fs = require('../core/fs.js');
 const hostedGitInfo = require('hosted-git-info');
 const utils = require('./utils.js');
-import {joinPath, runCmd} from 'embark-utils';
+import { joinPath, runCmd, errorMessage } from 'embark-utils';
 const semver = require('semver');
 const {promisify} = require('util');
 const {execSync} = require('child_process');
@@ -37,7 +37,7 @@ class TemplateGenerator {
     try {
       await promisify(utils.downloadFile)(url, tmpFilePath);
     } catch (e) {
-      console.error(utils.errorMessage(e).red);
+      console.error(errorMessage(e).red);
       throw e;
     }
   }
@@ -55,7 +55,7 @@ class TemplateGenerator {
     try {
       ext = await this.getExternalProject(uri);
     } catch (e) {
-      console.error(utils.errorMessage(e).red);
+      console.error(errorMessage(e).red);
       process.exit(1);
     }
     let {url, filePath, browse} = ext;

packages/embark/src/lib/utils/utils.js

@@ -256,34 +256,6 @@ function interceptLogs(consoleContext, logger) {
   };
 }
 
-function errorMessage(e) {
-  if (typeof e === 'string') {
-    return e;
-  } else if (e && e.message) {
-    return e.message;
-  }
-  return e;
-}
-
-function isFolder(node) {
-  return node.children && node.children.length;
-}
-
-function isNotFolder(node){
-  return !isFolder(node);
-}
-
-function byName(a, b) {
-  return a.name.localeCompare(b.name);
-}
-
-function fileTreeSort(nodes){
-  const folders = nodes.filter(isFolder).sort(byName);
-  const files = nodes.filter(isNotFolder).sort(byName);
-
-  return folders.concat(files);
-}
-
 function getWindowSize() {
   const windowSize = require('window-size');
   if (windowSize) {
@@ -336,8 +308,6 @@ module.exports = {
   normalizeInput,
   groupBy,
   interceptLogs,
-  errorMessage,
-  fileTreeSort,
   getWindowSize,
   isEs6Module,
   urlJoin