diff --git a/packages/embark-coverage/.npmrc b/packages/embark-coverage/.npmrc new file mode 100644 index 000000000..e031d3432 --- /dev/null +++ b/packages/embark-coverage/.npmrc @@ -0,0 +1,4 @@ +engine-strict = true +package-lock = false +save-exact = true +scripts-prepend-node-path = true diff --git a/packages/embark-coverage/README.md b/packages/embark-coverage/README.md new file mode 100644 index 000000000..64b3dfe14 --- /dev/null +++ b/packages/embark-coverage/README.md @@ -0,0 +1,6 @@ +# `embark-coverage` + +> Code Coverage capabilities for Embark + +Visit [embark.status.im](https://embark.status.im/) to get started with +[Embark](https://github.com/embark-framework/embark). diff --git a/packages/embark-coverage/package.json b/packages/embark-coverage/package.json new file mode 100644 index 000000000..032959bac --- /dev/null +++ b/packages/embark-coverage/package.json @@ -0,0 +1,69 @@ +{ + "name": "embark-coverage", + "version": "4.1.0-beta.1", + "author": "Iuri Matias ", + "contributors": [], + "description": "Code Coverage capabilities for Embark", + "homepage": "https://github.com/embark-framework/embark/tree/master/packages/embark-coverage#readme", + "bugs": "https://github.com/embark-framework/embark/issues", + "keywords": [ + "blockchain", + "dapps", + "ethereum", + "ipfs", + "serverless", + "solc", + "solidity" + ], + "files": [ + "dist" + ], + "license": "MIT", + "repository": { + "directory": "packages/embark-coverage", + "type": "git", + "url": "https://github.com/embark-framework/embark.git" + }, + "main": "./dist/index.js", + "scripts": { + "build": "cross-env BABEL_ENV=node babel src --extensions \".ts\" --out-dir dist --root-mode upward --source-maps", + "ci": "npm run qa", + "clean": "npm run reset", + "lint": "npm-run-all lint:*", + "lint:ts": "tslint -c tslint.json \"src/**/*.ts\"", + "package": "npm pack", + "qa": "npm-run-all lint typecheck build package", + "reset": "npx rimraf dist embark-*.tgz package", + "start": "npm run watch", + "typecheck": "tsc", + "watch": "run-p watch:*", + "watch:build": "npm run build -- --verbose --watch", + "watch:typecheck": "npm run typecheck -- --preserveWatchOutput --watch" + }, + "dependencies": { + "@babel/runtime-corejs2": "7.3.1", + "embark-core": "^4.1.0-beta.0", + "embark-utils": "^4.1.0-beta.0", + "fs-extra": "7.0.1", + "globule": "1.2.1", + "semver": "5.6.0", + "solidity-parser-antlr": "0.4.2", + "web3-eth-contract": "1.0.0-beta.37" + }, + "devDependencies": { + "@babel/cli": "7.2.3", + "@babel/core": "7.2.2", + "@types/web3": "1.0.12", + "cross-env": "5.2.0", + "eslint": "5.7.0", + "npm-run-all": "4.1.5", + "rimraf": "2.6.3", + "tslint": "5.16.0", + "typescript": "3.4.5" + }, + "engines": { + "node": ">=8.12.0", + "npm": ">=6.4.1", + "yarn": ">=1.12.3" + } +} diff --git a/packages/embark/src/lib/modules/coverage/contractEnhanced.ts b/packages/embark-coverage/src/contractEnhanced.ts similarity index 98% rename from packages/embark/src/lib/modules/coverage/contractEnhanced.ts rename to packages/embark-coverage/src/contractEnhanced.ts index 114d26a21..f3bd3131d 100644 --- a/packages/embark/src/lib/modules/coverage/contractEnhanced.ts +++ b/packages/embark-coverage/src/contractEnhanced.ts @@ -1,3 +1,5 @@ +import { File } from "embark-utils"; +import * as fs from "fs-extra"; import * as path from "path"; import parser, { LineColumn, Location } from "solidity-parser-antlr"; import { EventLog } from "web3/types"; @@ -9,9 +11,6 @@ import { InstrumentWalker } from "./instrumentWalker"; import { coverageContractsPath } from "./path"; import { BranchType, Coverage } from "./types"; -const File = require("../../core/file"); -const fs = require("../../core/fs"); - const STATEMENT_EVENT = "__StatementCoverage"; const POINT_FACTOR = 1000000000; diff --git a/packages/embark/src/lib/modules/coverage/eventId.ts b/packages/embark-coverage/src/eventId.ts similarity index 100% rename from packages/embark/src/lib/modules/coverage/eventId.ts rename to packages/embark-coverage/src/eventId.ts diff --git a/packages/embark/src/lib/modules/coverage/index.ts b/packages/embark-coverage/src/index.ts similarity index 96% rename from packages/embark/src/lib/modules/coverage/index.ts rename to packages/embark-coverage/src/index.ts index 88043c780..a48bba391 100644 --- a/packages/embark/src/lib/modules/coverage/index.ts +++ b/packages/embark-coverage/src/index.ts @@ -1,11 +1,9 @@ -import { dappPath } from "embark-utils"; +import { dappPath, File, removePureView } from "embark-utils"; import * as globule from "globule"; import * as path from "path"; import Web3Contract from "web3/eth/contract"; import { Contract, Embark } from "embark"; -import { File } from "../../core/file"; -import { removePureView } from "../../utils/solidity/code"; import { ContractEnhanced } from "./contractEnhanced"; import { coverageContractsPath } from "./path"; import { Coverage as ICoverage } from "./types"; diff --git a/packages/embark/src/lib/modules/coverage/injector.ts b/packages/embark-coverage/src/injector.ts similarity index 100% rename from packages/embark/src/lib/modules/coverage/injector.ts rename to packages/embark-coverage/src/injector.ts diff --git a/packages/embark/src/lib/modules/coverage/instrumentWalker.ts b/packages/embark-coverage/src/instrumentWalker.ts similarity index 100% rename from packages/embark/src/lib/modules/coverage/instrumentWalker.ts rename to packages/embark-coverage/src/instrumentWalker.ts diff --git a/packages/embark/src/lib/modules/coverage/instrumenter.ts b/packages/embark-coverage/src/instrumenter.ts similarity index 100% rename from packages/embark/src/lib/modules/coverage/instrumenter.ts rename to packages/embark-coverage/src/instrumenter.ts diff --git a/packages/embark/src/lib/modules/coverage/path.ts b/packages/embark-coverage/src/path.ts similarity index 76% rename from packages/embark/src/lib/modules/coverage/path.ts rename to packages/embark-coverage/src/path.ts index e7e8f4b99..8895b36c1 100644 --- a/packages/embark/src/lib/modules/coverage/path.ts +++ b/packages/embark-coverage/src/path.ts @@ -1,5 +1,3 @@ import * as path from "path"; -const fs = require("../../core/fs"); - export const coverageContractsPath = () => path.join("coverage", "instrumentedContracts"); diff --git a/packages/embark/src/lib/modules/coverage/types.ts b/packages/embark-coverage/src/types.ts similarity index 100% rename from packages/embark/src/lib/modules/coverage/types.ts rename to packages/embark-coverage/src/types.ts diff --git a/packages/embark-coverage/tsconfig.json b/packages/embark-coverage/tsconfig.json new file mode 100644 index 000000000..52d43eaaa --- /dev/null +++ b/packages/embark-coverage/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"] +} diff --git a/packages/embark-coverage/tslint.json b/packages/embark-coverage/tslint.json new file mode 100644 index 000000000..0946f2096 --- /dev/null +++ b/packages/embark-coverage/tslint.json @@ -0,0 +1,3 @@ +{ + "extends": "../../tslint.json" +} diff --git a/packages/embark-utils/index.d.ts b/packages/embark-utils/index.d.ts index 17ee6e7d3..72b59d0e2 100644 --- a/packages/embark-utils/index.d.ts +++ b/packages/embark-utils/index.d.ts @@ -1 +1,21 @@ -declare module "embark-utils"; +declare module "embark-utils" { + class File { + path: string; + constructor(options: any); + prepareForCompilation(isCoverage?: boolean): any; + } + + function compact(array: any): any; + function checkIsAvailable(url: string, callback: any): void; + function dockerHostSwap(host: string): string; + function dappPath(...names: string[]): string; + function escapeHtml(message: any): string; + function embarkPath(...names: string[]): string; + function exit(code?: any): void; + function findNextPort(port: number): Promise; + function jsonFunctionReplacer(key: any, value: any): any; + function fuzzySearch(text: string, list: any, filter: any): any; + function getExternalContractUrl(file: string, provideUrl: string): string; + function recursiveMerge(target: any, source: any): any; + function removePureView(dir: string): void; +} diff --git a/packages/embark-utils/package.json b/packages/embark-utils/package.json index b8387b74c..f86853650 100644 --- a/packages/embark-utils/package.json +++ b/packages/embark-utils/package.json @@ -52,6 +52,7 @@ "embark-i18n": "^4.1.0-beta.1", "ethereumjs-wallet": "0.6.3", "follow-redirects": "1.5.7", + "fs-extra": "7.0.1", "fuzzy": "0.1.3", "merge": "1.2.1", "multihashes": "0.4.14", diff --git a/packages/embark-utils/src/collections.ts b/packages/embark-utils/src/collections.ts index f59255e47..e0c33b600 100644 --- a/packages/embark-utils/src/collections.ts +++ b/packages/embark-utils/src/collections.ts @@ -11,3 +11,10 @@ export function recursiveMerge(target: any, source: any) { export function compact(array: any) { return array.filter((n: any) => n); } + +export function groupBy(array: any, key: any) { + return array.reduce((rv: any, x: any) => { + (rv[x[key]] = rv[x[key]] || []).push(x); + return rv; + }, {}); +} diff --git a/packages/embark-utils/src/file.ts b/packages/embark-utils/src/file.ts new file mode 100644 index 000000000..2050d2018 --- /dev/null +++ b/packages/embark-utils/src/file.ts @@ -0,0 +1,177 @@ +import { __ } from "embark-i18n"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { downloadFile } from "./network"; +import { dappPath, embarkPath } from "./pathUtils"; +import { ImportRemapping, prepareForCompilation } from "./solidity/remapImports"; + +const HTTP_CONTRACTS_DIRECTORY = ".embark/contracts/"; + +export enum Types { + embarkInternal = "embark_internal", + dappFile = "dapp_file", + custom = "custom", + http = "http", +} + +export class File { + public type: Types; + public externalUrl: string = ""; + public path = ""; + public basedir: string; + public resolver: (callback: (content: string) => void) => void; + public pluginPath: string; + public storageConfig: any; + public providerUrl: string; + public importRemappings: ImportRemapping[] = []; + public originalPath: string; + + constructor(options: any) { + this.type = options.type; + + this.basedir = options.basedir || ""; + this.resolver = options.resolver; + this.pluginPath = options.pluginPath ? options.pluginPath : ""; + this.storageConfig = options.storageConfig; + this.providerUrl = ""; + this.originalPath = options.originalPath || ""; + + if (this.type === Types.custom && this.pluginPath) { + this.path = path.join(this.pluginPath, options.path).replace(dappPath(), ""); + if (this.path.startsWith("/")) { + this.path = this.path.substring(1); + } + } else if (this.type === Types.http) { + const external = getExternalContractUrl(options.externalUrl, this.providerUrl); + if (external !== null) { + this.externalUrl = external.url; + this.path = path.normalize(dappPath(external.filePath)); + } + } else { + this.path = path.normalize(options.path); + } + } + + public async prepareForCompilation(isCoverage = false) { + if (!this.path.endsWith(".sol")) { + return Promise.reject(__("This method is only supported for Solidity files")); + } + return prepareForCompilation(this, isCoverage); + } + + public get content(): Promise { + return new Promise((resolve) => { + switch (this.type) { + case Types.embarkInternal: { + const content = fs.readFileSync(embarkPath(path.join("dist", this.path)), "utf-8"); + return resolve(content); + } + + case Types.dappFile: { + const content = fs.readFileSync(this.path, "utf-8").toString(); + return resolve(content); + } + + case Types.custom: { + return this.resolver((content: string) => { + resolve(content); + }); + } + + case Types.http: { + fs.ensureFileSync(this.path); + return downloadFile(this.externalUrl, this.path, () => { + const content = fs.readFileSync(this.path, "utf-8"); + resolve(content); + }); + } + } + }); + } + +} + +export function getExternalContractUrl(file: string, providerUrl: string) { + let url; + const RAW_URL = "https://raw.githubusercontent.com/"; + const DEFAULT_SWARM_GATEWAY = "https://swarm-gateways.net/"; + const MALFORMED_SWARM_ERROR = "Malformed Swarm gateway URL for "; + const MALFORMED_ERROR = "Malformed Github URL for "; + const MALFORMED_IPFS_ERROR = "Malformed IPFS URL for "; + const IPFS_GETURL_NOTAVAILABLE = "IPFS getUrl is not available. Please set it in your storage config. For more info: https://embark.status.im/docs/storage_configuration.html"; + if (file.startsWith("https://github")) { + const file_path = file.match(/https:\/\/github\.[a-z]+\/(.*)/); + if (!file_path) { + console.error(MALFORMED_ERROR + file); + return null; + } + url = `${RAW_URL}${file_path[1].replace("blob/", "")}`; + } else if (file.startsWith("ipfs")) { + if (!providerUrl) { + console.error(IPFS_GETURL_NOTAVAILABLE); + return null; + } + let file_path = file.match(/ipfs:\/\/([-a-zA-Z0-9]+)\/(.*)/); + if (!file_path) { + file_path = file.match(/ipfs:\/\/([-a-zA-Z0-9]+)/); + if (!file_path) { + console.error(MALFORMED_IPFS_ERROR + file); + return null; + } + } + let matchResult = file_path[1]; + if (file_path[2]) { + matchResult += "/" + file_path[2]; + } + url = `${providerUrl}${matchResult}`; + return { + filePath: HTTP_CONTRACTS_DIRECTORY + matchResult, + url, + }; + } else if (file.startsWith("git")) { + // Match values + // [0] entire input + // [1] git:// + // [2] user + // [3] repository + // [4] path + // [5] branch + const file_path = file.match( + /(git:\/\/)?github\.[a-z]+\/([-a-zA-Z0-9@:%_+.~#?&=]+)\/([-a-zA-Z0-9@:%_+.~#?&=]+)\/([-a-zA-Z0-9@:%_+.~?\/&=]+)#?([a-zA-Z0-9\/_.-]*)?/, + ); + if (!file_path) { + console.error(MALFORMED_ERROR + file); + return null; + } + let branch = file_path[5]; + if (!branch) { + branch = "master"; + } + url = `${RAW_URL}${file_path[2]}/${file_path[3]}/${branch}/${file_path[4]}`; + } else if (file.startsWith("http")) { + url = file; + } else if (file.startsWith("bzz")) { + if (!providerUrl) { + url = DEFAULT_SWARM_GATEWAY + file; + } else { + let file_path = file.match(/bzz:\/([-a-zA-Z0-9]+)\/(.*)/); + if (!file_path) { + file_path = file.match(/bzz:\/([-a-zA-Z0-9]+)/); + if (!file_path) { + console.log(MALFORMED_SWARM_ERROR + file); + return null; + } + } + url = providerUrl + "/" + file; + } + } else { + return null; + } + const match = url.match( + /\.[a-z]+\/([-a-zA-Z0-9@:%_+.~#?&\/=]+)/, + ); + return { + filePath: HTTP_CONTRACTS_DIRECTORY + (match !== null ? match[1] : ""), + url, + }; +} diff --git a/packages/embark-utils/src/index.js b/packages/embark-utils/src/index.js index 9f21947a0..c8bfe638e 100644 --- a/packages/embark-utils/src/index.js +++ b/packages/embark-utils/src/index.js @@ -1,12 +1,10 @@ -const path = require('path'); -const os = require('os'); const http = require('follow-redirects').http; const https = require('follow-redirects').https; const shelljs = require('shelljs'); const clipboardy = require('clipboardy'); const {canonicalHost, defaultCorsHost, defaultHost, dockerHostSwap, isDocker} = require('./host'); -const {findNextPort} = require('./network'); +const { findNextPort, downloadFile } = require('./network'); const logUtils = require('./log-utils'); const toposortGraph = require('./toposort'); import { unitRegex } from './constants'; @@ -17,6 +15,7 @@ import { hexToNumber, decodeParams, sha3, + sha512, isHex, soliditySha3, toChecksumAddress @@ -24,10 +23,14 @@ import { import { getAddressToContract, getTransactionParams } from './transactionUtils'; import LongRunningProcessTimer from './longRunningProcessTimer'; import AccountParser from './accountParser'; +import { dappPath, embarkPath, ipcPath, joinPath, tmpDir, urlJoin } from './pathUtils'; const { extendZeroAddressShorthand, replaceZeroAddressShorthand } = AddressUtils; -import { compact, last, recursiveMerge } from './collections'; +import { compact, last, recursiveMerge, groupBy } from './collections'; +import { prepareForCompilation } from './solidity/remapImports'; +import { removePureView } from './solidity/code'; +import { File, getExternalContractUrl, Types } from './file'; function timer(ms) { const then = Date.now(); @@ -60,15 +63,6 @@ function hashTo32ByteHexString(hash) { return '0x' + multihash.toHexString(digest); } -function sha512(arg) { - if (typeof arg !== 'string') { - throw new TypeError('argument must be a string'); - } - const crypto = require('crypto'); - const hash = crypto.createHash('sha512'); - return hash.update(arg).digest('hex'); -} - function exit(code) { process.exit(code); } @@ -239,12 +233,6 @@ function buildUrlFromConfig(configObj) { return buildUrl(configObj.protocol, canonicalHost(configObj.host), configObj.port, configObj.type); } -function joinPath() { - return path.join.apply(path.join, arguments); -} - -function tmpDir(...args) { return joinPath(os.tmpdir(), ...args); } - function errorMessage(e) { if (typeof e === 'string') { return e; @@ -254,32 +242,6 @@ function errorMessage(e) { return e; } -function dappPath(...names) { - const DAPP_PATH = process.env.DAPP_PATH || process.cwd(); - return path.join(DAPP_PATH, ...names); -} - -function ipcPath(basename, usePipePathOnWindows = false) { - if (!(basename && typeof basename === 'string')) { - throw new TypeError('first argument must be a non-empty string'); - } - if (process.platform === 'win32' && usePipePathOnWindows) { - return `\\\\.\\pipe\\${basename}`; - } - return joinPath( - tmpDir(`embark-${sha512(dappPath()).slice(0, 8)}`), - basename - ); -} - -function embarkPath(...names) { - const EMBARK_PATH = process.env.EMBARK_PATH; - if (!EMBARK_PATH) { - throw new Error('environment variable EMBARK_PATH was not set'); - } - return path.join(EMBARK_PATH, ...names); -} - const Utils = { buildUrl, @@ -288,6 +250,7 @@ const Utils = { tmpDir, ipcPath, dappPath, + downloadFile, embarkPath, jsonFunctionReplacer, fuzzySearch, @@ -305,6 +268,7 @@ const Utils = { getTransactionParams, isDocker, checkIsAvailable, + File, findNextPort, fileTreeSort, hashTo32ByteHexString, @@ -316,15 +280,21 @@ const Utils = { prepareContractsConfig, getWeiBalanceFromString, getHexBalanceFromString, + getExternalContractUrl, + groupBy, sha512, sha3, timer, + Types, unitRegex, + urlJoin, + removePureView, runCmd, escapeHtml: logUtils.escapeHtml, normalizeInput: logUtils.normalizeInput, LogHandler: require('./logHandler'), LongRunningProcessTimer, + prepareForCompilation, proposeAlternative, toChecksumAddress, toposort, diff --git a/packages/embark-utils/src/network.ts b/packages/embark-utils/src/network.ts index ccd0e0c83..9e2a74663 100644 --- a/packages/embark-utils/src/network.ts +++ b/packages/embark-utils/src/network.ts @@ -1,4 +1,7 @@ +import * as fs from "fs-extra"; import * as net from "net"; +const http = require("follow-redirects").http; +const https = require("follow-redirects").https; export function findNextPort(port: number) { const server = net.createServer(); @@ -8,3 +11,21 @@ export function findNextPort(port: number) { server.listen(port, () => server.close()); }); } + +export function downloadFile(url: string, dest: string, cb: any) { + const file = fs.createWriteStream(dest); + (url.substring(0, 5) === "https" ? https : http).get(url, (response: any) => { + if (response.statusCode !== 200) { + cb(`Download failed, response code ${response.statusCode}`); + return; + } + response.pipe(file); + file.on("finish", () => { + file.close(); + cb(); + }); + }).on("error", (err: Error) => { + fs.unlink(dest); + cb(err.message); + }); +} diff --git a/packages/embark-utils/src/pathUtils.js b/packages/embark-utils/src/pathUtils.js new file mode 100644 index 000000000..38b33fdcf --- /dev/null +++ b/packages/embark-utils/src/pathUtils.js @@ -0,0 +1,50 @@ +import * as path from 'path'; +import * as os from 'os'; +import { sha512 } from './web3Utils'; + +export function joinPath() { + return path.join.apply(path.join, arguments); +} + +export function tmpDir(...args) { return joinPath(os.tmpdir(), ...args); } + +export function dappPath(...names) { + return path.join(process.env.DAPP_PATH || process.cwd(), ...names); +} + +export function ipcPath(basename, usePipePathOnWindows = false) { + if (!(basename && typeof basename === 'string')) { + throw new TypeError('first argument must be a non-empty string'); + } + if (process.platform === 'win32' && usePipePathOnWindows) { + return `\\\\.\\pipe\\${basename}`; + } + return joinPath( + tmpDir(`embark-${sha512(dappPath()).slice(0, 8)}`), + basename + ); +} + +export function embarkPath(...names) { + const EMBARK_PATH = process.env.EMBARK_PATH; + if (!EMBARK_PATH) { + throw new Error('environment variable EMBARK_PATH was not set'); + } + return path.join(EMBARK_PATH, ...names); +} + +export function urlJoin(url, path) { + let urlChunks = url.split('/'); + let levels = path.split('../'); + + // remove relative path parts from end of url + urlChunks = urlChunks.slice(0, urlChunks.length - levels.length); + + // remove relative path parts from start of match + levels.splice(0, levels.length - 1); + + // add on our match so we can join later + urlChunks = urlChunks.concat(levels.join().replace('./', '')); + + return urlChunks.join('/'); +} diff --git a/packages/embark/src/lib/utils/solidity/code.ts b/packages/embark-utils/src/solidity/code.ts similarity index 92% rename from packages/embark/src/lib/utils/solidity/code.ts rename to packages/embark-utils/src/solidity/code.ts index 72d47f92d..9646cfbca 100644 --- a/packages/embark/src/lib/utils/solidity/code.ts +++ b/packages/embark-utils/src/solidity/code.ts @@ -1,8 +1,7 @@ +import * as fs from "fs-extra"; import * as globule from "globule"; import * as path from "path"; -const fs = require("../../core/fs"); - export const removePureView = (dir: string) => { globule.find(path.join(dir, "**/*.sol")).forEach((filepath) => { let source = fs.readFileSync(filepath, "utf-8"); diff --git a/packages/embark/src/lib/utils/solidity/remapImports.ts b/packages/embark-utils/src/solidity/remapImports.ts similarity index 96% rename from packages/embark/src/lib/utils/solidity/remapImports.ts rename to packages/embark-utils/src/solidity/remapImports.ts index 2cf7925e3..5da350865 100644 --- a/packages/embark/src/lib/utils/solidity/remapImports.ts +++ b/packages/embark-utils/src/solidity/remapImports.ts @@ -1,11 +1,10 @@ -import { dappPath, embarkPath } from "embark-utils"; +import * as fs from "fs-extra"; import * as path from "path"; -import { File, Types } from "../../core/file"; +import { groupBy } from "../collections"; +import { File, Types } from "../file"; +import { dappPath, embarkPath, urlJoin } from "../pathUtils"; import { removePureView, replacePureView } from "./code"; -const { urlJoin, groupBy } = require("../../utils/utils"); -const fs = require("../../core/fs"); - const FIND_IMPORTS_REGEX = /^import[\s]*(['"])(.*)\1;/gm; const FIND_FILE_REGEX = /import[\s]*(['"])(.*)\1;/; diff --git a/packages/embark-utils/src/web3Utils.ts b/packages/embark-utils/src/web3Utils.ts index a89330d40..5efe6b26b 100644 --- a/packages/embark-utils/src/web3Utils.ts +++ b/packages/embark-utils/src/web3Utils.ts @@ -44,6 +44,15 @@ export function sha3(arg: any) { return web3.utils.sha3(arg); } +export function sha512(arg: string) { + if (typeof arg !== "string") { + throw new TypeError("argument must be a string"); + } + const crypto = require("crypto"); + const hash = crypto.createHash("sha512"); + return hash.update(arg).digest("hex"); +} + export function isHex(hex: string) { return web3.utils.isHex(hex); } diff --git a/packages/embark/package.json b/packages/embark/package.json index 858944ede..da14d1a9e 100644 --- a/packages/embark/package.json +++ b/packages/embark/package.json @@ -102,6 +102,7 @@ "embark-console-listener": "^4.1.0-beta.1", "embark-contracts-manager": "^4.1.0-beta.1", "embark-core": "^4.1.0-beta.1", + "embark-coverage": "^4.1.0-beta.1", "embark-debugger": "^4.1.0-beta.1", "embark-deploy-tracker": "^4.1.0-beta.1", "embark-deployment": "^4.1.0-beta.1", diff --git a/packages/embark/src/lib/core/config.js b/packages/embark/src/lib/core/config.js index b2a9206e5..907315079 100644 --- a/packages/embark/src/lib/core/config.js +++ b/packages/embark/src/lib/core/config.js @@ -11,15 +11,17 @@ import { canonicalHost, dappPath, defaultHost, + File, + Types, recursiveMerge, AddressUtils, unitRegex, getWeiBalanceFromString, - prepareContractsConfig + prepareContractsConfig, + getExternalContractUrl } from 'embark-utils'; const cloneDeep = require('lodash.clonedeep'); const { replaceZeroAddressShorthand } = AddressUtils; -import { File, Types } from "./file"; const DEFAULT_CONFIG_PATH = 'config/'; const PACKAGE = require('../../../package.json'); @@ -404,7 +406,7 @@ Config.prototype.loadExternalContractsFiles = function() { let externalContractFile = null; if (contract.file.startsWith('http') || contract.file.startsWith('git') || contract.file.startsWith('ipfs') || contract.file.startsWith('bzz')) { - const fileObj = utils.getExternalContractUrl(contract.file, this.providerUrl); + const fileObj = getExternalContractUrl(contract.file, this.providerUrl); if (!fileObj) { return this.logger.error(__("HTTP contract file not found") + ": " + contract.file); } diff --git a/packages/embark/src/lib/core/engine.js b/packages/embark/src/lib/core/engine.js index 7a0232bb8..29cc4eecd 100644 --- a/packages/embark/src/lib/core/engine.js +++ b/packages/embark/src/lib/core/engine.js @@ -320,7 +320,7 @@ class Engine { } codeCoverageService(_options) { - this.registerModule('coverage'); + this.registerModulePackage('embark-coverage'); } testRunnerService(options) { diff --git a/packages/embark/src/lib/core/file.ts b/packages/embark/src/lib/core/file.ts deleted file mode 100644 index 95157637b..000000000 --- a/packages/embark/src/lib/core/file.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { __ } from "embark-i18n"; -import { dappPath, embarkPath } from "embark-utils"; -import * as path from "path"; -import { ImportRemapping, prepareForCompilation } from "../utils/solidity/remapImports"; - -const fs = require("./fs.js"); -const utils = require("../utils/utils"); - -export enum Types { - embarkInternal = "embark_internal", - dappFile = "dapp_file", - custom = "custom", - http = "http", -} - -export class File { - public type: Types; - public externalUrl: string = ""; - public path: string; - public basedir: string; - public resolver: (callback: (content: string) => void) => void; - public pluginPath: string; - public storageConfig: any; - public providerUrl: string; - public importRemappings: ImportRemapping[] = []; - public originalPath: string; - - constructor(options: any) { - this.type = options.type; - - this.basedir = options.basedir || ""; - this.resolver = options.resolver; - this.pluginPath = options.pluginPath ? options.pluginPath : ""; - this.storageConfig = options.storageConfig; - this.providerUrl = ""; - this.originalPath = options.originalPath || ""; - - if (this.type === Types.custom && this.pluginPath) { - this.path = path.join(this.pluginPath, options.path).replace(dappPath(), ""); - if (this.path.startsWith("/")) { - this.path = this.path.substring(1); - } - } else if (this.type === Types.http) { - const external = utils.getExternalContractUrl(options.externalUrl, this.providerUrl); - this.externalUrl = external.url; - this.path = path.normalize(dappPath(external.filePath)); - } else { - this.path = path.normalize(options.path); - } - } - - public async prepareForCompilation(isCoverage = false) { - if (!this.path.endsWith(".sol")) { - return Promise.reject(__("This method is only supported for Solidity files")); - } - return prepareForCompilation(this, isCoverage); - } - - public get content(): Promise { - return new Promise((resolve) => { - switch (this.type) { - case Types.embarkInternal: { - const content = fs.readFileSync(embarkPath(path.join("dist", this.path)), "utf-8"); - return resolve(content); - } - - case Types.dappFile: { - const content = fs.readFileSync(this.path, "utf-8").toString(); - return resolve(content); - } - - case Types.custom: { - return this.resolver((content: string) => { - resolve(content); - }); - } - - case Types.http: { - fs.ensureFileSync(this.path); - return utils.downloadFile(this.externalUrl, this.path, () => { - const content = fs.readFileSync(this.path, "utf-8"); - resolve(content); - }); - } - } - }); - } - -} diff --git a/packages/embark/src/lib/utils/template_generator.js b/packages/embark/src/lib/utils/template_generator.js index 18b7942dd..8206974aa 100644 --- a/packages/embark/src/lib/utils/template_generator.js +++ b/packages/embark/src/lib/utils/template_generator.js @@ -3,7 +3,7 @@ const findUp = require('find-up'); const fs = require('../core/fs.js'); const hostedGitInfo = require('hosted-git-info'); const utils = require('./utils.js'); -import { embarkPath, joinPath, runCmd, errorMessage } from 'embark-utils'; +import { embarkPath, downloadFile, joinPath, runCmd, errorMessage } from 'embark-utils'; const semver = require('semver'); const {promisify} = require('util'); const {execSync} = require('child_process'); @@ -34,7 +34,7 @@ class TemplateGenerator { console.log(__('Downloading template...').green); fs.mkdirpSync(utils.dirname(tmpFilePath)); try { - await promisify(utils.downloadFile)(url, tmpFilePath); + await promisify(downloadFile)(url, tmpFilePath); } catch (e) { console.error(errorMessage(e).red); throw e; diff --git a/packages/embark/src/lib/utils/utils.js b/packages/embark/src/lib/utils/utils.js index 9afa9fb59..3c0386463 100644 --- a/packages/embark/src/lib/utils/utils.js +++ b/packages/embark/src/lib/utils/utils.js @@ -78,24 +78,6 @@ function sed(file, pattern, replace) { shelljs.sed('-i', pattern, replace, file); } -function downloadFile(url, dest, cb) { - const o_fs = require('fs-extra'); - var file = o_fs.createWriteStream(dest); - (url.substring(0, 5) === 'https' ? https : http).get(url, function (response) { - if (response.statusCode !== 200) { - cb(`Download failed, response code ${response.statusCode}`); - return; - } - response.pipe(file); - file.on('finish', function () { - file.close(cb); - }); - }).on('error', function (err) { - o_fs.unlink(dest); - cb(err.message); - }); -} - function extractTar(filename, packageDirectory, cb) { const o_fs = require('fs-extra'); const tar = require('tar'); @@ -117,92 +99,6 @@ function extractZip(filename, packageDirectory, opts, cb) { }); } -function getExternalContractUrl(file,providerUrl) { - const constants = require('embark-core/constants'); - let url; - const RAW_URL = 'https://raw.githubusercontent.com/'; - const DEFAULT_SWARM_GATEWAY = 'https://swarm-gateways.net/'; - const MALFORMED_SWARM_ERROR = 'Malformed Swarm gateway URL for '; - const MALFORMED_ERROR = 'Malformed Github URL for '; - const MALFORMED_IPFS_ERROR = 'Malformed IPFS URL for '; - const IPFS_GETURL_NOTAVAILABLE = 'IPFS getUrl is not available. Please set it in your storage config. For more info: https://embark.status.im/docs/storage_configuration.html'; - if (file.startsWith('https://github')) { - const match = file.match(/https:\/\/github\.[a-z]+\/(.*)/); - if (!match) { - console.error(MALFORMED_ERROR + file); - return null; - } - url = `${RAW_URL}${match[1].replace('blob/', '')}`; - } else if (file.startsWith('ipfs')) { - if(!providerUrl) { - console.error(IPFS_GETURL_NOTAVAILABLE); - return null; - } - let match = file.match(/ipfs:\/\/([-a-zA-Z0-9]+)\/(.*)/); - if(!match) { - match = file.match(/ipfs:\/\/([-a-zA-Z0-9]+)/); - if(!match) { - console.error(MALFORMED_IPFS_ERROR + file); - return null; - } - } - let matchResult = match[1]; - if(match[2]) { - matchResult += '/' + match[2]; - } - url = `${providerUrl}${matchResult}`; - return { - url, - filePath: constants.httpContractsDirectory + matchResult - }; - } else if (file.startsWith('git')) { - // Match values - // [0] entire input - // [1] git:// - // [2] user - // [3] repository - // [4] path - // [5] branch - const match = file.match( - /(git:\/\/)?github\.[a-z]+\/([-a-zA-Z0-9@:%_+.~#?&=]+)\/([-a-zA-Z0-9@:%_+.~#?&=]+)\/([-a-zA-Z0-9@:%_+.~?\/&=]+)#?([a-zA-Z0-9\/_.-]*)?/ - ); - if (!match) { - console.error(MALFORMED_ERROR + file); - return null; - } - let branch = match[5]; - if (!branch) { - branch = 'master'; - } - url = `${RAW_URL}${match[2]}/${match[3]}/${branch}/${match[4]}`; - } else if (file.startsWith('http')) { - url = file; - } else if(file.startsWith('bzz')){ - if(!providerUrl) { - url = DEFAULT_SWARM_GATEWAY + file; - } else { - let match = file.match(/bzz:\/([-a-zA-Z0-9]+)\/(.*)/); - if(!match){ - match = file.match(/bzz:\/([-a-zA-Z0-9]+)/); - if(!match){ - console.log(MALFORMED_SWARM_ERROR + file); - return null; - } - } - url = providerUrl + '/' + file; - } - } else { - return null; - } - const match = url.match( - /\.[a-z]+\/([-a-zA-Z0-9@:%_+.~#?&\/=]+)/ - ); - return { - url, - filePath: constants.httpContractsDirectory + match[1] - }; -} - function isValidDomain(v) { // from: https://github.com/miguelmota/is-valid-domain if (typeof v !== 'string') return false; @@ -224,13 +120,6 @@ function isValidDomain(v) { return isValid; } -function groupBy(array, key) { - return array.reduce(function (rv, x) { - (rv[x[key]] = rv[x[key]] || []).push(x); - return rv; - }, {}); -} - function interceptLogs(consoleContext, logger) { let context = {}; context.console = consoleContext; @@ -273,22 +162,6 @@ function isEs6Module(module) { return (typeof module === 'function' && isConstructor(module)) || (typeof module === 'object' && typeof module.default === 'function' && module.__esModule); } -function urlJoin(url, path) { - let urlChunks = url.split('/'); - let levels = path.split('../'); - - // remove relative path parts from end of url - urlChunks = urlChunks.slice(0, urlChunks.length - levels.length); - - // remove relative path parts from start of match - levels.splice(0, levels.length - 1); - - // add on our match so we can join later - urlChunks = urlChunks.concat(levels.join().replace('./', '')); - - return urlChunks.join('/'); -} - module.exports = { dirname, filesMatchingPattern, @@ -301,14 +174,10 @@ module.exports = { isValidDomain, cd, sed, - downloadFile, extractTar, extractZip, - getExternalContractUrl, normalizeInput, - groupBy, interceptLogs, getWindowSize, - isEs6Module, - urlJoin + isEs6Module }; diff --git a/packages/embark/src/test/contracts.js b/packages/embark/src/test/contracts.js index eb82f9b8c..05d0d95f3 100644 --- a/packages/embark/src/test/contracts.js +++ b/packages/embark/src/test/contracts.js @@ -1,5 +1,5 @@ /*global describe, it, require*/ -import { File, Types } from "../lib/core/file"; +import { File, Types } from "embark-utils"; let ContractsManager = require('embark-contracts-manager'); let Compiler = require('embark-compiler'); diff --git a/packages/embark/src/test/events.js b/packages/embark/src/test/events.js index d34d8a6d1..b6e2dc6e0 100644 --- a/packages/embark/src/test/events.js +++ b/packages/embark/src/test/events.js @@ -1,5 +1,5 @@ /*globals describe, it, before, beforeEach*/ -const {File, Types} = require("../lib/core/file"); +import { File, Types } from 'embark-utils'; const Assert = require("assert"); const {expect} = require("chai"); const fs = require("../lib/core/fs"); diff --git a/packages/embark/src/test/file.js b/packages/embark/src/test/file.js index d495a2fcd..6e49f36ed 100644 --- a/packages/embark/src/test/file.js +++ b/packages/embark/src/test/file.js @@ -1,6 +1,5 @@ /*globals describe, it*/ -const { dappPath } = require('embark-utils'); -const {File, Types} = require("../lib/core/file"); +const { dappPath, File, Types } = require('embark-utils'); const {expect} = require("chai"); const fs = require("../lib/core/fs"); diff --git a/packages/embark/src/test/modules/compiler/compiler.js b/packages/embark/src/test/modules/compiler/compiler.js index e4db0c225..d2a772a9a 100644 --- a/packages/embark/src/test/modules/compiler/compiler.js +++ b/packages/embark/src/test/modules/compiler/compiler.js @@ -1,5 +1,5 @@ /*global describe, it, require*/ -import { File, Types } from "../../../lib/core/file"; +import { File, Types } from "embark-utils"; const assert = require('assert'); diff --git a/packages/embark/src/test/modules/solidity/remapImports.js b/packages/embark/src/test/modules/solidity/remapImports.js index ab1a9a5a9..63f213d8d 100644 --- a/packages/embark/src/test/modules/solidity/remapImports.js +++ b/packages/embark/src/test/modules/solidity/remapImports.js @@ -1,8 +1,6 @@ /*globals describe, it, before*/ -const { dappPath } = require('embark-utils'); -const {File, Types} = require("../../../lib/core/file"); +const { dappPath, File, Types, prepareForCompilation } = require('embark-utils'); const path = require("path"); -const remapImports = require("../../../lib/utils/solidity/remapImports"); const {expect} = require("chai"); const fs = require("../../../lib/core/fs"); const fsNode = require("fs"); @@ -13,7 +11,7 @@ describe('embark.RemapImports', function () { describe('Import remappings from local file', function () { before('do the remappings', async () => { file = new File({path: 'contracts/recursive_test_0.sol', type: Types.dappFile}); - content = await remapImports.prepareForCompilation(file); + content = await prepareForCompilation(file); }); it("should find and add remappings for all recursive imports", (done) => { @@ -64,7 +62,7 @@ describe('embark.RemapImports', function () { describe('Import remappings from external URL', function () { before('do the external HTTP contract remappings', async () => { file = new File({externalUrl: 'https://github.com/embark-framework/embark/master/packages/embark/src/test/contracts/recursive_test_0.sol', type: Types.http}); - content = await remapImports.prepareForCompilation(file); + content = await prepareForCompilation(file); }); it("should find and add remappings for all recursive imports", (done) => { diff --git a/packages/embark/src/test/modules/solidity/solidity.js b/packages/embark/src/test/modules/solidity/solidity.js index 9d9c085ae..051776862 100644 --- a/packages/embark/src/test/modules/solidity/solidity.js +++ b/packages/embark/src/test/modules/solidity/solidity.js @@ -1,7 +1,7 @@ /*global describe, it, require*/ -import { File, Types } from "../../../lib/core/file.js"; const fs = require('../../../lib/core/fs'); import { IPC } from 'embark-core'; +import { File, Types } from 'embark-utils'; let SolidityCompiler = require('embark-solidity'); let TestLogger = require('../../../lib/utils/test_logger'); diff --git a/packages/embark/src/test/utils.js b/packages/embark/src/test/utils.js index f25e3ff05..252ec41b9 100644 --- a/packages/embark/src/test/utils.js +++ b/packages/embark/src/test/utils.js @@ -1,12 +1,13 @@ /*global describe, it*/ const Utils = require('../lib/utils/utils'); +import { getExternalContractUrl } from 'embark-utils'; const assert = require('assert'); const constants = require('embark-core/constants'); describe('embark.utils', function () { describe('#getExternalContractUrl', function () { it('should get the right url for a https://github file', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'https://github.com/embark-framework/embark/blob/master/dapps/templates/demo/contracts/simple_storage.sol' ); assert.deepEqual(fileObj, @@ -17,14 +18,14 @@ describe('embark.utils', function () { }); it('should fail for a malformed https://github file', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'https://github/embark-framework/embark/blob/master/dapps/templates/demo/contracts/simple_storage.sol' ); assert.strictEqual(fileObj, null); }); it('should get the right url for a git:// file with no branch #', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'git://github.com/status-im/contracts/contracts/identity/ERC725.sol' ); assert.deepEqual(fileObj, @@ -35,7 +36,7 @@ describe('embark.utils', function () { }); it('should get the right url for a git:// file with a branch #', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'git://github.com/status-im/contracts/contracts/identity/ERC725.sol#myBranch' ); assert.deepEqual(fileObj, @@ -46,14 +47,14 @@ describe('embark.utils', function () { }); it('should fail when the git:// file is malformed', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'git://github.com/identity/ERC725.sol#myBranch' ); assert.strictEqual(fileObj, null); }); it('should get the right url with a github.com file without branch #', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'github.com/status-im/contracts/contracts/identity/ERC725.sol' ); assert.deepEqual(fileObj, @@ -64,7 +65,7 @@ describe('embark.utils', function () { }); it('should get the right url with a github.com file with branch #', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'github.com/status-im/contracts/contracts/identity/ERC725.sol#theBranch' ); assert.deepEqual(fileObj, @@ -75,14 +76,14 @@ describe('embark.utils', function () { }); it('should fail with a malformed github.com url', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'github/status-im/contracts/contracts/identity/ERC725.sol#theBranch' ); assert.strictEqual(fileObj, null); }); it('should succeed with a generic http url', function () { - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( 'http://myurl.com/myFile.sol' ); assert.deepEqual(fileObj, { @@ -93,7 +94,7 @@ describe('embark.utils', function () { it('should get the correct default url for a correct bzz:/ swarm file', function () { const swarmFile = 'bzz:/someensdomain.eth/ERC725.sol'; - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( swarmFile ); assert.deepEqual(fileObj, { @@ -104,7 +105,7 @@ describe('embark.utils', function () { it('should get the correct url for a correct bzz:/ swarm file when a http swarm gateway is explicitly provided', function () { const swarmFile = 'bzz:/someensdomain.eth/ERC725.sol'; - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( swarmFile, 'http://localhost:8500' ); @@ -116,7 +117,7 @@ describe('embark.utils', function () { it('should get the correct url for a correct bzz:/ swarm file when a https swarm gateway is provided', function () { const swarmFile = 'bzz:/1ffe993abc835f480f688d07ad75ad1dbdbd1ddb368a08b7ed4d3e400771dd63'; - const fileObj = Utils.getExternalContractUrl( + const fileObj = getExternalContractUrl( swarmFile, 'https://swarm-gateways.net' ); diff --git a/tsconfig.json b/tsconfig.json index 06c44eab0..6e965dd7a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { + "allowJs": true, "esModuleInterop": true, - "isolatedModules": true, "lib": ["ES2017"], "module": "CommonJS", "noEmit": true,