mirror of
https://github.com/status-im/sourcecred.git
synced 2025-01-15 23:25:35 +00:00
418b745d7c
Summary:
In this newly added module, we load the structural state of a git
repository into memory. We do not load into memory the contents of any
blobs, so this is not enough information to perform any analysis
requiring file diffing. However, it is sufficient to develop a notion of
“this file was changed in this commit”, by simply diffing the trees.
Test Plan:
Unit tests added; `yarn test` suffices. Reading these snapshots is
pretty easy, even though they’re filled with hashes:
- First, read over the commit specifications on lines 69–83 of
`loadRepository.test.js`, so you know what to expect.
- In the snapshot file, keep handy the time-ordered list of commit
SHAs at the bottom of the file, so that you know which commit SHA is
which.
- To verify that the large snapshot is correct: for each commit, read
the corresponding tree object and make sure that the structure is
correct.
- To verify the small snapshot, just check that it’s the correct
subset of the large snapshot.
- If you want to verify that the SHA for a blob is correct, open a
terminal and run `git hash-object -t blob --stdin`; then, enter the
content of the blob and press `<C-d>`. The result is the blob SHA.
To run a sanity-check on a large repository: apply the following patch:
<details>
<summary>Patch to print out statistics about loaded repository</summary>
```diff
diff --git a/config/paths.js b/config/paths.js
index d2f25fb..8fa2023 100644
--- a/config/paths.js
+++ b/config/paths.js
@@ -62,5 +62,6 @@ module.exports = {
fetchAndPrintGithubRepo: resolveApp(
"src/plugins/github/bin/fetchAndPrintGithubRepo.js"
),
+ loadRepository: resolveApp("src/plugins/git/loadRepository.js"),
},
};
diff --git a/src/plugins/git/loadRepository.js b/src/plugins/git/loadRepository.js
index a76b66c..9380941 100644
--- a/src/plugins/git/loadRepository.js
+++ b/src/plugins/git/loadRepository.js
@@ -106,3 +106,7 @@ function findTrees(git: GitDriver, rootTrees: Set<Hash>): Tree[] {
}
return result;
}
+
+const result = loadRepository(...process.argv.slice(2));
+console.log("commits", result.commits.size);
+console.log("trees", result.trees.size);
```
</details>
Then, run `yarn backend` and put the following script in `test.sh`:
<details>
<summary>Contents for `test.sh`</summary>
```shell
#!/bin/bash
set -eu
repo="$1"
ref="$2"
via_node() {
node bin/loadRepository.js "${repo}" "${ref}"
}
via_git() (
cd "${repo}"
printf 'commits '
git rev-list "${ref}" | wc -l
printf 'trees '
git rev-list "${ref}" |
while read -r commit; do
git rev-parse "${commit}^{tree}"
git ls-tree -rt "${commit}" \
| grep ' tree ' \
| cut -f 1 | cut -d ' ' -f 3
done | sort | uniq | wc -l
)
echo
printf 'Running directly via git...\n'
time a="$(via_git)"
echo
printf 'Running Node script...\n'
time b="$(via_node)"
diff -u <(cat <<<"${a}") <(cat <<<"${b}")
```
</details>
Finally, run `./test.sh /path/to/some/repo origin/master`, and verify
that it exits successfully (zero diff). Here are some timing results on
SourceCred and TensorBoard:
- SourceCred: 0.973s via Node, 0.327s via git.
- TensorBoard: 30.836s via Node, 6.895s via git.
For TensorFlow, running via git takes 7m33.995s. Running via Node fails
with an out-of-memory error after 39 minutes, with 10GB RAM and 4GB
swap. See details below.
<details>
<summary>
Full timing details, commit SHAs, and OOM error message
</summary>
```
+ ./test.sh /home/wchargin/git/sourcecred 01634aabcca3756b38e13aaf2f451cfbda2ad5ea
Running directly via git...
real 0m0.327s
user 0m0.016s
sys 0m0.052s
Running Node script...
real 0m0.973s
user 0m0.268s
sys 0m0.176s
+ ./test.sh /home/wchargin/git/tensorboard 7aa1ab9d60671056b8811b7099eec08650f2e4fd
Running directly via git...
real 0m6.895s
user 0m0.600s
sys 0m0.832s
Running Node script...
real 0m30.836s
user 0m3.216s
sys 0m10.588s
+ ./test.sh /home/wchargin/git/tensorflow 968addadfd4e4f5688eedc31f92a9066329ff6a7
Running directly via git...
real 7m33.995s
user 5m21.124s
sys 1m5.476s
Running Node script...
FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
1: node::Abort() [node]
2: 0x121a2cc [node]
3: v8::Utils::ReportOOMFailure(char const*, bool) [node]
4: v8::internal::V8::FatalProcessOutOfMemory(char const*, bool) [node]
5: v8::internal::Factory::NewFixedArray(int, v8::internal::PretenureFlag) [node]
6: v8::internal::DeoptimizationInputData::New(v8::internal::Isolate*, int, v8::internal::PretenureFlag) [node]
7: v8::internal::compiler::CodeGenerator::PopulateDeoptimizationData(v8::internal::Handle<v8::internal::Code>) [node]
8: v8::internal::compiler::CodeGenerator::FinalizeCode() [node]
9: v8::internal::compiler::PipelineImpl::FinalizeCode() [node]
10: v8::internal::compiler::PipelineCompilationJob::FinalizeJobImpl() [node]
11: v8::internal::Compiler::FinalizeCompilationJob(v8::internal::CompilationJob*) [node]
12: v8::internal::OptimizingCompileDispatcher::InstallOptimizedFunctions() [node]
13: v8::internal::Runtime_TryInstallOptimizedCode(int, v8::internal::Object**, v8::internal::Isolate*) [node]
14: 0x12dc8b08463d
```
</details>
wchargin-branch: load-git-repositories
# Please enter the commit message for your changes. Lines starting
# with '#' will be kept; you may remove them yourself if you want to.
# An empty message aborts the commit.
#
# Date: Mon Apr 23 23:02:14 2018 -0700
#
# HEAD detached at origin/wchargin-load-git-repositories
# Changes to be committed:
# modified: package.json
# new file: src/plugins/git/__snapshots__/loadRepository.test.js.snap
# new file: src/plugins/git/loadRepository.js
# new file: src/plugins/git/loadRepository.test.js
#
# Untracked files:
# out
# runtests.sh
# src/plugins/artifact/editor/ArtifactSetInput.js
# src/plugins/git/repository.js
# test.sh
# todo
#
124 lines
3.3 KiB
JSON
124 lines
3.3 KiB
JSON
{
|
|
"name": "sourcecred-explorer",
|
|
"version": "0.1.0",
|
|
"private": true,
|
|
"dependencies": {
|
|
"aphrodite": "^2.1.0",
|
|
"autoprefixer": "7.1.6",
|
|
"babel-core": "6.26.0",
|
|
"babel-eslint": "7.2.3",
|
|
"babel-jest": "20.0.3",
|
|
"babel-loader": "7.1.2",
|
|
"babel-preset-react-app": "^3.1.1",
|
|
"babel-runtime": "6.26.0",
|
|
"case-sensitive-paths-webpack-plugin": "2.1.1",
|
|
"chalk": "1.1.3",
|
|
"css-loader": "0.28.7",
|
|
"dotenv": "4.0.0",
|
|
"dotenv-expand": "4.0.1",
|
|
"eslint": "4.10.0",
|
|
"eslint-config-react-app": "^2.1.0",
|
|
"eslint-loader": "1.9.0",
|
|
"eslint-plugin-flowtype": "2.39.1",
|
|
"eslint-plugin-import": "2.8.0",
|
|
"eslint-plugin-jsx-a11y": "5.1.1",
|
|
"eslint-plugin-react": "7.4.0",
|
|
"extract-text-webpack-plugin": "3.0.2",
|
|
"file-loader": "1.1.5",
|
|
"flow-bin": "^0.65.0",
|
|
"fs-extra": "3.0.1",
|
|
"html-webpack-plugin": "2.29.0",
|
|
"husky": "^0.14.3",
|
|
"isomorphic-fetch": "^2.2.1",
|
|
"jest": "20.0.4",
|
|
"json-stable-stringify": "^1.0.1",
|
|
"lint-staged": "^6.1.1",
|
|
"lodash.isequal": "^4.5.0",
|
|
"object-assign": "4.1.1",
|
|
"postcss-flexbugs-fixes": "3.2.0",
|
|
"postcss-loader": "2.0.8",
|
|
"prettier": "^1.10.2",
|
|
"promise": "8.0.1",
|
|
"raf": "3.4.0",
|
|
"react": "^16.2.0",
|
|
"react-dev-utils": "^5.0.0",
|
|
"react-dom": "^16.2.0",
|
|
"style-loader": "0.19.0",
|
|
"sw-precache-webpack-plugin": "0.11.4",
|
|
"url-loader": "0.6.2",
|
|
"webpack": "3.8.1",
|
|
"webpack-dev-server": "2.9.4",
|
|
"webpack-manifest-plugin": "1.3.2",
|
|
"whatwg-fetch": "2.0.3"
|
|
},
|
|
"scripts": {
|
|
"precommit": "npm run lint-staged",
|
|
"lint-staged": "lint-staged",
|
|
"prettify": "prettier --write '**/*.js'",
|
|
"check-pretty": "prettier --list-different '**/*.js'",
|
|
"start": "node scripts/start.js",
|
|
"build": "node scripts/build.js",
|
|
"backend": "node scripts/backend.js",
|
|
"test": "node scripts/test.js --env=jsdom",
|
|
"flow": "flow",
|
|
"travis": "npm run check-pretty && npm run flow && CI=true npm run test"
|
|
},
|
|
"license": "MIT",
|
|
"lint-staged": {
|
|
"**/*.js": "prettier --list-different"
|
|
},
|
|
"jest": {
|
|
"collectCoverageFrom": [
|
|
"src/**/*.{js,jsx,mjs}"
|
|
],
|
|
"setupFiles": [
|
|
"<rootDir>/config/polyfills.js"
|
|
],
|
|
"testMatch": [
|
|
"<rootDir>/src/**/__tests__/**/*.{js,jsx,mjs}",
|
|
"<rootDir>/src/**/?(*.)(spec|test).{js,jsx,mjs}"
|
|
],
|
|
"testEnvironment": "node",
|
|
"testURL": "http://localhost",
|
|
"transform": {
|
|
"^.+\\.(js|jsx|mjs)$": "<rootDir>/node_modules/babel-jest",
|
|
"^.+\\.css$": "<rootDir>/config/jest/cssTransform.js",
|
|
"^(?!.*\\.(js|jsx|mjs|css|json)$)": "<rootDir>/config/jest/fileTransform.js"
|
|
},
|
|
"transformIgnorePatterns": [
|
|
"[/\\\\]node_modules[/\\\\].+\\.(js|jsx|mjs)$"
|
|
],
|
|
"moduleNameMapper": {
|
|
"^react-native$": "react-native-web"
|
|
},
|
|
"moduleFileExtensions": [
|
|
"web.js",
|
|
"mjs",
|
|
"js",
|
|
"json",
|
|
"web.jsx",
|
|
"jsx",
|
|
"node"
|
|
]
|
|
},
|
|
"babel": {
|
|
"plugins": [
|
|
"flow-react-proptypes"
|
|
],
|
|
"presets": [
|
|
"react-app"
|
|
]
|
|
},
|
|
"eslintConfig": {
|
|
"extends": "react-app"
|
|
},
|
|
"devDependencies": {
|
|
"babel-plugin-flow-react-proptypes": "^18.0.0",
|
|
"enzyme": "^3.3.0",
|
|
"enzyme-adapter-react-16": "^1.1.1",
|
|
"enzyme-to-json": "^3.3.3",
|
|
"mkdirp": "^0.5.1",
|
|
"tmp": "^0.0.33"
|
|
}
|
|
}
|