mirror of
https://github.com/status-im/sourcecred.git
synced 2025-03-01 04:30:29 +00:00
7f81337d74
Summary: We store the relational view in `view.json.gz` instead of `view.json`, taking advantage of the isomorphic `pako` library for gzip encoding and decoding. Sample space savings (note that post bodies are included; i.e., #747 has not been applied): SAVE OLD (B) NEW (B) REPO 89.7% 25326 2617 sourcecred/example-github 82.9% 3257576 555948 sourcecred/sourcecred 85.2% 11287621 1665884 ipfs/js-ipfs 88.0% 20953425 2520358 gitcoinco/web 84.4% 38196825 5951459 ipfs/go-ipfs 84.9% 205770642 31101452 tensorflow/tensorflow <details> <summary>Script to generate space savings output</summary> ```shell savings() { printf '% 7s % 11s % 11s %s\n' 'SAVE' 'OLD (B)' 'NEW (B)' 'REPO' for repo; do file="${SOURCECRED_DIRECTORY}/data/${repo}/github/view.json.gz" if ! [ -f "${file}" ]; then printf >&2 'warn: no such file %s\n' "${file}" continue fi script="$(sed -e 's/^ *//' <<EOF repo = '${repo}' pre_size = $(<"${file}" gzip -dc | wc -c) post_size = $(<"${file}" wc -c) percentage = '%0.1f%%' % (100 * (1 - post_size / pre_size)) p = '% 7s % 11d % 11d %s' % (percentage, pre_size, post_size, repo) print(p) EOF )" python3 -c "${script}" done } ``` </details> Closes #750. Test Plan: Comparing the raw old version with the decompressed new version shows that they are identical: ``` $ <~/tmp/sourcecred/data/sourcecred/example-github/github/view.json \ > shasum -a 256 - 63853b9d3f918274aafacf5198787e18185a61b9c95faf640a1e61f5d11fa19f - $ <~/tmp/sourcecred/data/sourcecred/example-github/github/view.json.gz \ > gzip -dc | shasum -a 256 63853b9d3f918274aafacf5198787e18185a61b9c95faf640a1e61f5d11fa19f - ``` Additionally, `yarn test --full` passes, and `yarn start` still loads data and runs PageRank properly. wchargin-branch: gzip-relational-view
142 lines
4.0 KiB
JSON
142 lines
4.0 KiB
JSON
{
|
|
"name": "sourcecred",
|
|
"version": "0.1.0",
|
|
"private": true,
|
|
"dependencies": {
|
|
"@oclif/command": "^1.4.36",
|
|
"@oclif/config": "^1.6.33",
|
|
"@oclif/plugin-help": "^2.0.5",
|
|
"aphrodite": "^2.1.0",
|
|
"chalk": "1.1.3",
|
|
"commonmark": "^0.28.1",
|
|
"express": "^4.16.3",
|
|
"fs-extra": "3.0.1",
|
|
"history": "^3.0.0",
|
|
"isomorphic-fetch": "^2.2.1",
|
|
"json-stable-stringify": "^1.0.1",
|
|
"lodash.clonedeep": "^4.5.0",
|
|
"lodash.isequal": "^4.5.0",
|
|
"lodash.sortby": "^4.7.0",
|
|
"mkdirp": "^0.5.1",
|
|
"object-assign": "4.1.1",
|
|
"pako": "^1.0.6",
|
|
"promise": "8.0.1",
|
|
"react": "^16.4.1",
|
|
"react-dom": "^16.4.1",
|
|
"react-router": "3.2.1",
|
|
"retry": "^0.12.0",
|
|
"tmp": "^0.0.33",
|
|
"whatwg-fetch": "2.0.3"
|
|
},
|
|
"devDependencies": {
|
|
"autoprefixer": "7.1.6",
|
|
"babel-core": "6.26.0",
|
|
"babel-eslint": "7.2.3",
|
|
"babel-jest": "20.0.3",
|
|
"babel-loader": "7.1.2",
|
|
"babel-plugin-transform-es2015-for-of": "^6.23.0",
|
|
"babel-preset-react-app": "^3.1.1",
|
|
"babel-runtime": "6.26.0",
|
|
"case-sensitive-paths-webpack-plugin": "2.1.1",
|
|
"copy-webpack-plugin": "^4.5.2",
|
|
"css-loader": "0.28.7",
|
|
"dotenv": "4.0.0",
|
|
"dotenv-expand": "4.0.1",
|
|
"enzyme": "^3.3.0",
|
|
"enzyme-adapter-react-16": "^1.1.1",
|
|
"enzyme-to-json": "^3.3.3",
|
|
"eslint": "4.10.0",
|
|
"eslint-config-react-app": "^2.1.0",
|
|
"eslint-loader": "1.9.0",
|
|
"eslint-plugin-flowtype": "2.39.1",
|
|
"eslint-plugin-import": "2.8.0",
|
|
"eslint-plugin-jsx-a11y": "5.1.1",
|
|
"eslint-plugin-react": "7.4.0",
|
|
"extract-text-webpack-plugin": "3.0.2",
|
|
"file-loader": "1.1.5",
|
|
"flow-bin": "^0.77.0",
|
|
"html-webpack-plugin": "2.29.0",
|
|
"jest": "23.4.1",
|
|
"jest-fetch-mock": "^1.6.5",
|
|
"postcss-flexbugs-fixes": "3.2.0",
|
|
"postcss-loader": "2.0.8",
|
|
"prettier": "^1.13.4",
|
|
"raf": "3.4.0",
|
|
"react-dev-utils": "^5.0.0",
|
|
"rimraf": "^2.6.2",
|
|
"static-site-generator-webpack-plugin": "^3.4.1",
|
|
"style-loader": "0.19.0",
|
|
"url-loader": "0.6.2",
|
|
"webpack": "3.8.1",
|
|
"webpack-dev-server": "2.9.4",
|
|
"webpack-manifest-plugin": "1.3.2",
|
|
"webpack-node-externals": "^1.7.2"
|
|
},
|
|
"scripts": {
|
|
"prettify": "prettier --write '**/*.js'",
|
|
"check-pretty": "prettier --list-different '**/*.js'",
|
|
"start": "NODE_ENV=development webpack-dev-server --config config/makeWebpackConfig.js",
|
|
"build": "NODE_ENV=production webpack --config config/makeWebpackConfig.js",
|
|
"backend": "node scripts/backend.js",
|
|
"test": "node ./config/test.js",
|
|
"unit": "BABEL_ENV=test NODE_ENV=test jest --env=jsdom",
|
|
"sharness": "make -sC ./sharness prove PROVE_OPTS=-f TEST_OPTS='--chain-lint'",
|
|
"sharness-full": "make -sC ./sharness prove PROVE_OPTS=-f TEST_OPTS='--chain-lint --long'",
|
|
"coverage": "npm run unit -- --coverage",
|
|
"flow": "flow",
|
|
"lint": "eslint src config --max-warnings 0"
|
|
},
|
|
"license": "MIT",
|
|
"jest": {
|
|
"collectCoverageFrom": [
|
|
"src/**/*.{js,jsx,mjs}"
|
|
],
|
|
"setupFiles": [
|
|
"<rootDir>/config/polyfills.js",
|
|
"<rootDir>/config/jest/setupJest.js"
|
|
],
|
|
"testMatch": [
|
|
"<rootDir>/src/**/__tests__/**/*.{js,jsx,mjs}",
|
|
"<rootDir>/src/**/?(*.)(spec|test).{js,jsx,mjs}"
|
|
],
|
|
"testEnvironment": "node",
|
|
"testURL": "http://localhost",
|
|
"transform": {
|
|
"^.+\\.(js|jsx|mjs)$": "<rootDir>/node_modules/babel-jest",
|
|
"^.+\\.css$": "<rootDir>/config/jest/cssTransform.js",
|
|
"^(?!.*\\.(js|jsx|mjs|css|json)$)": "<rootDir>/config/jest/fileTransform.js"
|
|
},
|
|
"transformIgnorePatterns": [
|
|
"[/\\\\]node_modules[/\\\\].+\\.(js|jsx|mjs)$"
|
|
],
|
|
"moduleNameMapper": {
|
|
"^react-native$": "react-native-web"
|
|
},
|
|
"moduleFileExtensions": [
|
|
"web.js",
|
|
"mjs",
|
|
"js",
|
|
"json",
|
|
"web.jsx",
|
|
"jsx",
|
|
"node"
|
|
]
|
|
},
|
|
"babel": {
|
|
"presets": [
|
|
"./config/babel"
|
|
]
|
|
},
|
|
"files": [
|
|
"/bin",
|
|
"/build"
|
|
],
|
|
"oclif": {
|
|
"commands": "./bin/commands",
|
|
"bin": "sourcecred",
|
|
"plugins": [
|
|
"@oclif/plugin-help"
|
|
]
|
|
}
|
|
}
|