status-im
/
embark-area-51
mirror of https://github.com/status-im/embark-area-51.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

add code to parse all files to check for http imports

This commit is contained in:
Jonathan Rainville 2018-04-20 09:52:13 -04:00
parent 9a79df68d9
commit abc89b2015
7 changed files with 238 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
lib/core/config.js
View File

@ -143,49 +143,6 @@ Config.prototype.loadContractsConfigFile = function() {
this.contractsConfig = this._mergeConfig(configFilePath, configObject, this.env);
};
Config.prototype.getExternalContractUrl = function (contract) {
let url;
const RAW_URL = 'https://raw.githubusercontent.com/';
const MALFORMED_ERROR = 'Malformed Github URL for ';
if (contract.file.startsWith('https://github')) {
const match = contract.file.match(/https:\/\/github\.[a-z]+\/(.*)/);
if (!match) {
this.logger.error(MALFORMED_ERROR + contract.file);
return null;
}
url = `${RAW_URL}${match[1].replace('blob/', '')}`;
} else if (contract.file.startsWith('git')) {
// Match values
// [0] entire input
// [1] git://
// [2] user
// [3] repository
// [4] path
// [5] branch
const match = contract.file.match(
/(git:\/\/)?github\.[a-z]+\/([a-zA-Z0-9_\-.]+)\/([a-zA-Z0-9_\-]+)\/([a-zA-Z0-9_\-\/.]+)#?([a-zA-Z0-1_\-.]*)?/
);
if (!match) {
this.logger.error(MALFORMED_ERROR + contract.file);
return null;
}
let branch = match[5];
if (!branch) {
branch = 'master';
}
url = `${RAW_URL}${match[2]}/${match[3]}/${branch}/${match[4]}`;
} else {
url = contract.file;
}
const match = url.match(
/\.[a-z]+\/([a-zA-Z0-9_\-\/.]+)/
);
return {
url,
filePath: match[1]
};
};
Config.prototype.loadExternalContractsFiles = function() {
let contracts = this.contractsConfig.contracts;
for (let contractName in contracts) {
@ -194,11 +151,11 @@ Config.prototype.loadExternalContractsFiles = function() {
continue;
}
if (contract.file.startsWith('http') || contract.file.startsWith('git')) {
const fileObj = this.getExternalContractUrl(contract);
const fileObj = utils.getExternalContractUrl(contract.file);
if (!fileObj) {
return this.logger.error("HTTP contract file not found: " + contract.file);
}
const localFile = constants.httpContractsDirectory + fileObj.filePath;
const localFile = fileObj.filePath;
this.contractsFiles.push(new File({filename: localFile, type: File.types.http, basedir: '', path: fileObj.url}));
} else if (fs.existsSync(contract.file)) {
this.contractsFiles.push(new File({filename: contract.file, type: File.types.dapp_file, basedir: '', path: contract.file}));

49
lib/core/file.js
View File

@ -2,6 +2,7 @@ const async = require('async');
const fs = require('./fs.js');
const path = require('path');
const request = require('request');
const utils = require('../utils/utils');
class File {
@ -13,21 +14,34 @@ class File {
this.resolver = options.resolver;
}
parseFileForImport(content, callback) {
parseFileForImport(content, isHttpContract, callback) {
if (typeof isHttpContract === 'function') {
callback = isHttpContract;
isHttpContract = false;
}
const self = this;
if (self.filename.indexOf('.sol') < 0) {
// Only supported in Solidity
return callback();
}
const regex = /import "([a-zA-Z0-9_\-.\\\/]+)";/g;
const regex = /import "([a-zA-Z0-9_\-.\\\/:]+)";/g;
let matches;
const filesToDownload = [];
const pathWithoutFile = path.dirname(self.path);
while ((matches = regex.exec(content))) {
filesToDownload.push({
const httpFileObj = utils.getExternalContractUrl(matches[1]);
const fileObj = {
fileRelativePath: path.join(path.dirname(self.filename), matches[1]),
url: `${pathWithoutFile}/${matches[1]}`
});
};
if (httpFileObj) {
fileObj.fileRelativePath = httpFileObj.filePath;
fileObj.url = httpFileObj.url;
} else if (!isHttpContract) {
// Just a normal import
continue;
}
filesToDownload.push(fileObj);
}
async.each(filesToDownload, ((fileObj, eachCb) => {
@ -63,7 +77,7 @@ class File {
fs.readFile(filename, next);
},
function parseForImports(content, next) {
self.parseFileForImport(content, (err) => {
self.parseFileForImport(content, true, (err) => {
next(err, content);
});
}
@ -77,17 +91,27 @@ class File {
}
content (callback) {
let content;
if (this.type === File.types.embark_internal) {
return callback(fs.readFileSync(fs.embarkPath(this.path)).toString());
content = fs.readFileSync(fs.embarkPath(this.path)).toString();
} else if (this.type === File.types.dapp_file) {
return callback(fs.readFileSync(this.path).toString());
content = fs.readFileSync(this.path).toString();
} else if (this.type === File.types.custom) {
return this.resolver(callback);
return this.resolver((theContent) => {
if (!this.parsedImports) {
this.parsedImports = true;
return this.parseFileForImport(content, () => {
callback(theContent);
});
}
callback(theContent);
});
} else if (this.type === File.types.http) {
this.downloadFile(this.filename, this.path, (content) => {
return this.downloadFile(this.filename, this.path, (content) => {
if (!content) {
return callback(content);
}
this.parsedImports = true;
this.path = this.filename;
this.type = File.types.dapp_file;
callback(content);
@ -95,6 +119,13 @@ class File {
} else {
throw new Error("unknown file: " + this.filename);
}
if (!this.parsedImports) {
this.parsedImports = true;
return this.parseFileForImport(content, () => {
callback(content);
});
}
callback(content);
}
}

49
lib/utils/utils.js
View File

@ -6,6 +6,7 @@ let https = require('follow-redirects').https;
let shelljs = require('shelljs');
var tar = require('tar');
var propose = require('propose');
const constants = require('../constants');
//let fs = require('../core/fs.js');
let o_fs = require('fs-extra');
@ -127,6 +128,51 @@ function pwd() {
return process.env.PWD || process.cwd();
}
function getExternalContractUrl(file) {
let url;
const RAW_URL = 'https://raw.githubusercontent.com/';
const MALFORMED_ERROR = 'Malformed Github URL for ';
if (file.startsWith('https://github')) {
const match = file.match(/https:\/\/github\.[a-z]+\/(.*)/);
if (!match) {
console.error(MALFORMED_ERROR + file);
return null;
}
url = `${RAW_URL}${match[1].replace('blob/', '')}`;
} else if (file.startsWith('git')) {
// Match values
// [0] entire input
// [1] git://
// [2] user
// [3] repository
// [4] path
// [5] branch
const match = file.match(
/(git:\/\/)?github\.[a-z]+\/([a-zA-Z0-9_\-.]+)\/([a-zA-Z0-9_\-]+)\/([a-zA-Z0-9_\-\/.]+)#?([a-zA-Z0-1_\-.]*)?/
);
if (!match) {
console.error(MALFORMED_ERROR + file);
return null;
}
let branch = match[5];
if (!branch) {
branch = 'master';
}
url = `${RAW_URL}${match[2]}/${match[3]}/${branch}/${match[4]}`;
} else if (file.startsWith('http')) {
url = file;
} else {
return null;
}
const match = url.match(
/\.[a-z]+\/([a-zA-Z0-9_\-\/.]+)/
);
return {
url,
filePath: constants.httpContractsDirectory + match[1]
};
}
module.exports = {
joinPath: joinPath,
filesMatchingPattern: filesMatchingPattern,
@ -143,5 +189,6 @@ module.exports = {
downloadFile: downloadFile,
extractTar: extractTar,
proposeAlternative: proposeAlternative,
pwd: pwd
pwd: pwd,
getExternalContractUrl
};

88
test/config.js
View File

@ -57,94 +57,6 @@ describe('embark.Config', function () {
});
});
describe('#getExternalContractUrl', function () {
it('should get the right url for a https://github file', function () {
const fileObj = config.getExternalContractUrl(
{file: 'https://github.com/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'}
);
assert.deepEqual(fileObj,
{
filePath: 'embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
url: 'https://raw.githubusercontent.com/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol'
});
});
it('should fail for a malformed https://github file', function () {
const fileObj = config.getExternalContractUrl(
{file: 'https://github/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'}
);
assert.strictEqual(fileObj, null);
});
it('should get the right url for a git:// file with no branch #', function () {
const fileObj = config.getExternalContractUrl(
{file: 'git://github.com/status-im/contracts/contracts/identity/ERC725.sol'}
);
assert.deepEqual(fileObj,
{
filePath: 'status-im/contracts/master/contracts/identity/ERC725.sol',
url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
});
});
it('should get the right url for a git:// file with a branch #', function () {
const fileObj = config.getExternalContractUrl(
{file: 'git://github.com/status-im/contracts/contracts/identity/ERC725.sol#myBranch'}
);
assert.deepEqual(fileObj,
{
filePath: 'status-im/contracts/myBranch/contracts/identity/ERC725.sol',
url: 'https://raw.githubusercontent.com/status-im/contracts/myBranch/contracts/identity/ERC725.sol'
});
});
it('should fail when the git:// file is malformed', function () {
const fileObj = config.getExternalContractUrl(
{file: 'git://github.com/identity/ERC725.sol#myBranch'}
);
assert.strictEqual(fileObj, null);
});
it('should get the right url with a github.com file without branch #', function () {
const fileObj = config.getExternalContractUrl(
{file: 'github.com/status-im/contracts/contracts/identity/ERC725.sol'}
);
assert.deepEqual(fileObj,
{
filePath: 'status-im/contracts/master/contracts/identity/ERC725.sol',
url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
});
});
it('should get the right url with a github.com file with branch #', function () {
const fileObj = config.getExternalContractUrl(
{file: 'github.com/status-im/contracts/contracts/identity/ERC725.sol#theBranch'}
);
assert.deepEqual(fileObj,
{
filePath: 'status-im/contracts/theBranch/contracts/identity/ERC725.sol',
url: 'https://raw.githubusercontent.com/status-im/contracts/theBranch/contracts/identity/ERC725.sol'
});
});
it('should fail with a malformed github.com url', function () {
const fileObj = config.getExternalContractUrl(
{file: 'github/status-im/contracts/contracts/identity/ERC725.sol#theBranch'}
);
assert.strictEqual(fileObj, null);
});
it('should succeed with a generic http url', function () {
const fileObj = config.getExternalContractUrl(
{file: 'http://myurl.com/myFile.sol'}
);
assert.deepEqual(fileObj, {
filePath: 'myFile.sol',
url: 'http://myurl.com/myFile.sol'
});
});
});
describe('#loadExternalContractsFiles', function () {
it('should create the right list of files and download', function () {
config.contractsFiles = [];

19
test/contracts/contract_with_http_import.sol Normal file
View File

@ -0,0 +1,19 @@
pragma solidity ^0.4.7;
contract SimpleStorage {
uint public storedData;
import "https://github.com/embark-framework/embark/blob/develop/test_apps/contracts_app/contracts/contract_args.sol";
function SimpleStorage(uint initialValue) {
storedData = initialValue;
}
function set(uint x) {
storedData = x;
}
function get() constant returns (uint retVal) {
return storedData;
}
}

36
test/file.js
View File

@ -16,7 +16,7 @@ describe('embark.File', function () {
cb();
});
file.parseFileForImport(contract, () => {
file.parseFileForImport(contract, true, () => {
assert.strictEqual(downloadFileStub.callCount, 1);
assert.strictEqual(downloadFileStub.firstCall.args[0],
path.normalize('.embark/contracts/embark-framework/embark/master/test_app/app/contracts/ownable.sol'));
@ -25,5 +25,39 @@ describe('embark.File', function () {
done();
});
});
it('should find all the imports but not call download because not a http contract', function (done) {
const contract = fs.readFileSync('./test/contracts/contract_with_import.sol').toString();
const file = new File({filename: '.embark/contracts/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
path: 'https://raw.githubusercontent.com/embark-framework/embark/develop/test_apps/test_app/app/contracts/simple_storage.sol'});
const downloadFileStub = sinon.stub(file, 'downloadFile')
.callsFake((path, url, cb) => {
cb();
});
file.parseFileForImport(contract, () => {
assert.strictEqual(downloadFileStub.callCount, 0);
done();
});
});
it('should find all the imports and call downlaod because it is an http import', function (done) {
const contract = fs.readFileSync('./test/contracts/contract_with_http_import.sol').toString();
const file = new File({filename: '.embark/contracts/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
path: 'https://raw.githubusercontent.com/embark-framework/embark/develop/test_apps/test_app/app/contracts/simple_storage.sol'});
const downloadFileStub = sinon.stub(file, 'downloadFile')
.callsFake((path, url, cb) => {
cb();
});
file.parseFileForImport(contract, () => {
assert.strictEqual(downloadFileStub.callCount, 1);
assert.strictEqual(downloadFileStub.firstCall.args[0],
'.embark/contracts/embark-framework/embark/develop/test_apps/contracts_app/contracts/contract_args.sol');
assert.strictEqual(downloadFileStub.firstCall.args[1],
'https://raw.githubusercontent.com/embark-framework/embark/develop/test_apps/contracts_app/contracts/contract_args.sol');
done();
});
});
});
});

94
test/test.js Normal file
View File

@ -0,0 +1,94 @@
/*global describe, it*/
const Utils = require('../lib/utils/utils');
const assert = require('assert');
const constants = require('../lib/constants');
describe('embark.utils', function () {
describe('#getExternalContractUrl', function () {
it('should get the right url for a https://github file', function () {
const fileObj = Utils.getExternalContractUrl(
'https://github.com/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'
);
assert.deepEqual(fileObj,
{
filePath: constants.httpContractsDirectory + 'embark-framework/embark/master/test_app/app/contracts/simple_storage.sol',
url: 'https://raw.githubusercontent.com/embark-framework/embark/master/test_app/app/contracts/simple_storage.sol'
});
});
it('should fail for a malformed https://github file', function () {
const fileObj = Utils.getExternalContractUrl(
'https://github/embark-framework/embark/blob/master/test_app/app/contracts/simple_storage.sol'
);
assert.strictEqual(fileObj, null);
});
it('should get the right url for a git:// file with no branch #', function () {
const fileObj = Utils.getExternalContractUrl(
'git://github.com/status-im/contracts/contracts/identity/ERC725.sol'
);
assert.deepEqual(fileObj,
{
filePath: constants.httpContractsDirectory + 'status-im/contracts/master/contracts/identity/ERC725.sol',
url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
});
});
it('should get the right url for a git:// file with a branch #', function () {
const fileObj = Utils.getExternalContractUrl(
'git://github.com/status-im/contracts/contracts/identity/ERC725.sol#myBranch'
);
assert.deepEqual(fileObj,
{
filePath: constants.httpContractsDirectory + 'status-im/contracts/myBranch/contracts/identity/ERC725.sol',
url: 'https://raw.githubusercontent.com/status-im/contracts/myBranch/contracts/identity/ERC725.sol'
});
});
it('should fail when the git:// file is malformed', function () {
const fileObj = Utils.getExternalContractUrl(
'git://github.com/identity/ERC725.sol#myBranch'
);
assert.strictEqual(fileObj, null);
});
it('should get the right url with a github.com file without branch #', function () {
const fileObj = Utils.getExternalContractUrl(
'github.com/status-im/contracts/contracts/identity/ERC725.sol'
);
assert.deepEqual(fileObj,
{
filePath: constants.httpContractsDirectory + 'status-im/contracts/master/contracts/identity/ERC725.sol',
url: 'https://raw.githubusercontent.com/status-im/contracts/master/contracts/identity/ERC725.sol'
});
});
it('should get the right url with a github.com file with branch #', function () {
const fileObj = Utils.getExternalContractUrl(
'github.com/status-im/contracts/contracts/identity/ERC725.sol#theBranch'
);
assert.deepEqual(fileObj,
{
filePath: constants.httpContractsDirectory + 'status-im/contracts/theBranch/contracts/identity/ERC725.sol',
url: 'https://raw.githubusercontent.com/status-im/contracts/theBranch/contracts/identity/ERC725.sol'
});
});
it('should fail with a malformed github.com url', function () {
const fileObj = Utils.getExternalContractUrl(
'github/status-im/contracts/contracts/identity/ERC725.sol#theBranch'
);
assert.strictEqual(fileObj, null);
});
it('should succeed with a generic http url', function () {
const fileObj = Utils.getExternalContractUrl(
'http://myurl.com/myFile.sol'
);
assert.deepEqual(fileObj, {
filePath: constants.httpContractsDirectory + 'myFile.sol',
url: 'http://myurl.com/myFile.sol'
});
});
});
});