From 7efcc136185a715be0e7262a39d823e7d4044bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Thu, 11 Apr 2019 21:21:29 +0200 Subject: [PATCH] Automatically run pagerank on `sourcecred load` (#1115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit updates the `sourcecred load` command so that it also automatically runs PageRank on completion. The implementation is slightly hacky, in that it prints two sets of task status headers/footers to console, for reasons described in a comment in the source code. The user-visible effect of this hack can be seen below: ``` ❯ node bin/sourcecred.js load sourcecred/example-github Starting tasks GO load-git GO load-github DONE load-github DONE load-git Overview Final result: SUCCESS Starting tasks GO run-pagerank DONE run-pagerank Overview Final result: SUCCESS ``` It would be good to clean this up, but for now I think it's acceptable. Note that it is not safe to assume that a PagerankGraph always exists for repos that are included in the RepoIdRegistry. The repo gets added to the registry before the pagerank task runs. Consumers that are loading the `PagerankGraph` can just check that the file exists, though. Test plan: I've added unit tests that verify that the right tasks are generated. Most importantly, the snapshot of the results of `sourcecred load` now include a snapshotted pagerank graph. (The snapshot was updated via `UPDATE_SNAPSHOT=1 yarn test --full`.) Further progress on #967. --- .../example-github/pagerankGraph.json | 1 + src/cli/load.js | 47 +++++++++++++---- src/cli/load.test.js | 50 ++++++++++++++++--- 3 files changed, 80 insertions(+), 18 deletions(-) create mode 100644 sharness/__snapshots__/example-github-load/data/sourcecred/example-github/pagerankGraph.json diff --git a/sharness/__snapshots__/example-github-load/data/sourcecred/example-github/pagerankGraph.json b/sharness/__snapshots__/example-github-load/data/sourcecred/example-github/pagerankGraph.json new file mode 100644 index 0000000..002c9c8 --- /dev/null +++ b/sharness/__snapshots__/example-github-load/data/sourcecred/example-github/pagerankGraph.json @@ -0,0 +1 @@ +[{"type":"sourcecred/pagerankGraph","version":"0.1.0"},{"froWeights":[2,2,2,2,2,2,0.25,0.25,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,1,1,1,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0.25,0,4,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"graphJSON":[{"type":"sourcecred/graph","version":"0.4.0"},{"edges":[{"address":["sourcecred","git","HAS_PARENT","2","COMMIT","0a223346b4e6dec0127b1e6aa892c4ee0424b66a","2","COMMIT","ec91adb718a6045b492303f00d8e8beb957dc780"],"dstIndex":4,"srcIndex":0},{"address":["sourcecred","git","HAS_PARENT","2","COMMIT","0a223346b4e6dec0127b1e6aa892c4ee0424b66a","2","COMMIT","ecc889dc94cf6da17ae6eab5bb7b7155f577519d"],"dstIndex":5,"srcIndex":0},{"address":["sourcecred","git","HAS_PARENT","2","COMMIT","6bd1b4c0b719c22c688a74863be07a699b7b9b34","2","COMMIT","c430bd74455105f77215ece51945094ceeee6c86"],"dstIndex":3,"srcIndex":1},{"address":["sourcecred","git","HAS_PARENT","2","COMMIT","6d5b3aa31ebb68a06ceb46bbd6cf49b6ccd6f5e6","2","COMMIT","0a223346b4e6dec0127b1e6aa892c4ee0424b66a"],"dstIndex":0,"srcIndex":2},{"address":["sourcecred","git","HAS_PARENT","2","COMMIT","c430bd74455105f77215ece51945094ceeee6c86","2","COMMIT","6d5b3aa31ebb68a06ceb46bbd6cf49b6ccd6f5e6"],"dstIndex":2,"srcIndex":3},{"address":["sourcecred","git","HAS_PARENT","2","COMMIT","ecc889dc94cf6da17ae6eab5bb7b7155f577519d","2","COMMIT","ec91adb718a6045b492303f00d8e8beb957dc780"],"dstIndex":4,"srcIndex":5},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","BOT","credbot","4","sourcecred","git","COMMIT","c430bd74455105f77215ece51945094ceeee6c86"],"dstIndex":3,"srcIndex":41},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","BOT","credbot","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","417104047"],"dstIndex":21,"srcIndex":41},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","4","sourcecred","git","COMMIT","0a223346b4e6dec0127b1e6aa892c4ee0424b66a"],"dstIndex":0,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","4","sourcecred","git","COMMIT","6d5b3aa31ebb68a06ceb46bbd6cf49b6ccd6f5e6"],"dstIndex":2,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","4","sourcecred","git","COMMIT","ec91adb718a6045b492303f00d8e8beb957dc780"],"dstIndex":4,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","4","sourcecred","git","COMMIT","ecc889dc94cf6da17ae6eab5bb7b7155f577519d"],"dstIndex":5,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","1"],"dstIndex":25,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","10"],"dstIndex":26,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","12"],"dstIndex":28,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","13"],"dstIndex":29,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","2"],"dstIndex":30,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","4"],"dstIndex":31,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","6"],"dstIndex":32,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","7"],"dstIndex":33,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","8"],"dstIndex":34,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","PULL","sourcecred","example-github","3"],"dstIndex":35,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","PULL","sourcecred","example-github","5"],"dstIndex":36,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","PULL","sourcecred","example-github","9"],"dstIndex":37,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813621"],"dstIndex":9,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","373768703"],"dstIndex":10,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","373768850"],"dstIndex":11,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576185"],"dstIndex":12,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576220"],"dstIndex":13,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576248"],"dstIndex":14,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576273"],"dstIndex":15,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576920"],"dstIndex":16,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576936"],"dstIndex":17,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","373768442"],"dstIndex":18,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","373768538"],"dstIndex":19,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","385223316"],"dstIndex":20,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","PULL","sourcecred","example-github","3","369162222"],"dstIndex":22,"srcIndex":42},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","wchargin","6","sourcecred","github","ISSUE","sourcecred","example-github","10"],"dstIndex":26,"srcIndex":43},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","wchargin","6","sourcecred","github","ISSUE","sourcecred","example-github","11"],"dstIndex":27,"srcIndex":43},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","wchargin","6","sourcecred","github","PULL","sourcecred","example-github","9"],"dstIndex":37,"srcIndex":43},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","wchargin","7","sourcecred","github","REVIEW","sourcecred","example-github","5","100313899"],"dstIndex":39,"srcIndex":43},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","wchargin","7","sourcecred","github","REVIEW","sourcecred","example-github","5","100314038"],"dstIndex":40,"srcIndex":43},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","wchargin","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813013"],"dstIndex":7,"srcIndex":43},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","wchargin","8","sourcecred","github","COMMENT","PULL","sourcecred","example-github","5","396430464"],"dstIndex":23,"srcIndex":43},{"address":["sourcecred","github","AUTHORS","5","sourcecred","github","USERLIKE","USER","wchargin","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":43},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","1"],"dstIndex":38,"srcIndex":25},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","10"],"dstIndex":38,"srcIndex":26},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","11"],"dstIndex":38,"srcIndex":27},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","12"],"dstIndex":38,"srcIndex":28},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","13"],"dstIndex":38,"srcIndex":29},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","2"],"dstIndex":38,"srcIndex":30},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","4"],"dstIndex":38,"srcIndex":31},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","6"],"dstIndex":38,"srcIndex":32},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","7"],"dstIndex":38,"srcIndex":33},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","ISSUE","sourcecred","example-github","8"],"dstIndex":38,"srcIndex":34},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","PULL","sourcecred","example-github","3"],"dstIndex":38,"srcIndex":35},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","PULL","sourcecred","example-github","5"],"dstIndex":38,"srcIndex":36},{"address":["sourcecred","github","HAS_PARENT","6","sourcecred","github","PULL","sourcecred","example-github","9"],"dstIndex":38,"srcIndex":37},{"address":["sourcecred","github","HAS_PARENT","7","sourcecred","github","REVIEW","sourcecred","example-github","5","100313899"],"dstIndex":36,"srcIndex":39},{"address":["sourcecred","github","HAS_PARENT","7","sourcecred","github","REVIEW","sourcecred","example-github","5","100314038"],"dstIndex":36,"srcIndex":40},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420811872"],"dstIndex":27,"srcIndex":6},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813013"],"dstIndex":27,"srcIndex":7},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813206"],"dstIndex":27,"srcIndex":8},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813621"],"dstIndex":27,"srcIndex":9},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","373768703"],"dstIndex":30,"srcIndex":10},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","373768850"],"dstIndex":30,"srcIndex":11},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576185"],"dstIndex":30,"srcIndex":12},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576220"],"dstIndex":30,"srcIndex":13},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576248"],"dstIndex":30,"srcIndex":14},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576273"],"dstIndex":30,"srcIndex":15},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576920"],"dstIndex":30,"srcIndex":16},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576936"],"dstIndex":30,"srcIndex":17},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","373768442"],"dstIndex":32,"srcIndex":18},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","373768538"],"dstIndex":32,"srcIndex":19},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","385223316"],"dstIndex":32,"srcIndex":20},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","417104047"],"dstIndex":32,"srcIndex":21},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","PULL","sourcecred","example-github","3","369162222"],"dstIndex":35,"srcIndex":22},{"address":["sourcecred","github","HAS_PARENT","8","sourcecred","github","COMMENT","PULL","sourcecred","example-github","5","396430464"],"dstIndex":36,"srcIndex":23},{"address":["sourcecred","github","HAS_PARENT","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":39,"srcIndex":24},{"address":["sourcecred","github","MENTIONS_AUTHOR","6","sourcecred","github","PULL","sourcecred","example-github","5","8","sourcecred","github","COMMENT","PULL","sourcecred","example-github","5","396430464","5","sourcecred","github","USERLIKE","USER","wchargin"],"dstIndex":23,"srcIndex":36},{"address":["sourcecred","github","MERGED_AS","6","sourcecred","github","PULL","sourcecred","example-github","3"],"dstIndex":0,"srcIndex":35},{"address":["sourcecred","github","MERGED_AS","6","sourcecred","github","PULL","sourcecred","example-github","5"],"dstIndex":2,"srcIndex":36},{"address":["sourcecred","github","REACTS","HEART","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","1"],"dstIndex":25,"srcIndex":42},{"address":["sourcecred","github","REACTS","HEART","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","13"],"dstIndex":29,"srcIndex":42},{"address":["sourcecred","github","REACTS","HEART","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","PULL","sourcecred","example-github","9"],"dstIndex":37,"srcIndex":42},{"address":["sourcecred","github","REACTS","HEART","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","PULL","sourcecred","example-github","5","396430464"],"dstIndex":23,"srcIndex":42},{"address":["sourcecred","github","REACTS","HEART","5","sourcecred","github","USERLIKE","USER","decentralion","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":42},{"address":["sourcecred","github","REACTS","HEART","5","sourcecred","github","USERLIKE","USER","wchargin","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":43},{"address":["sourcecred","github","REACTS","HOORAY","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","13"],"dstIndex":29,"srcIndex":42},{"address":["sourcecred","github","REACTS","HOORAY","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813206"],"dstIndex":8,"srcIndex":42},{"address":["sourcecred","github","REACTS","HOORAY","5","sourcecred","github","USERLIKE","USER","decentralion","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":42},{"address":["sourcecred","github","REACTS","HOORAY","5","sourcecred","github","USERLIKE","USER","wchargin","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":43},{"address":["sourcecred","github","REACTS","ROCKET","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","13"],"dstIndex":29,"srcIndex":42},{"address":["sourcecred","github","REACTS","THUMBS_UP","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","12"],"dstIndex":28,"srcIndex":42},{"address":["sourcecred","github","REACTS","THUMBS_UP","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","ISSUE","sourcecred","example-github","13"],"dstIndex":29,"srcIndex":42},{"address":["sourcecred","github","REACTS","THUMBS_UP","5","sourcecred","github","USERLIKE","USER","decentralion","6","sourcecred","github","PULL","sourcecred","example-github","9"],"dstIndex":37,"srcIndex":42},{"address":["sourcecred","github","REACTS","THUMBS_UP","5","sourcecred","github","USERLIKE","USER","decentralion","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813206"],"dstIndex":8,"srcIndex":42},{"address":["sourcecred","github","REACTS","THUMBS_UP","5","sourcecred","github","USERLIKE","USER","decentralion","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":42},{"address":["sourcecred","github","REACTS","THUMBS_UP","5","sourcecred","github","USERLIKE","USER","wchargin","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":43},{"address":["sourcecred","github","REFERENCES","4","sourcecred","git","COMMIT","0a223346b4e6dec0127b1e6aa892c4ee0424b66a","6","sourcecred","github","PULL","sourcecred","example-github","3"],"dstIndex":35,"srcIndex":0},{"address":["sourcecred","github","REFERENCES","4","sourcecred","git","COMMIT","6bd1b4c0b719c22c688a74863be07a699b7b9b34","5","sourcecred","github","USERLIKE","USER","wchargin"],"dstIndex":43,"srcIndex":1},{"address":["sourcecred","github","REFERENCES","4","sourcecred","git","COMMIT","6d5b3aa31ebb68a06ceb46bbd6cf49b6ccd6f5e6","6","sourcecred","github","PULL","sourcecred","example-github","5"],"dstIndex":36,"srcIndex":2},{"address":["sourcecred","github","REFERENCES","4","sourcecred","git","COMMIT","c430bd74455105f77215ece51945094ceeee6c86","5","sourcecred","github","USERLIKE","USER","wchargin"],"dstIndex":43,"srcIndex":3},{"address":["sourcecred","github","REFERENCES","6","sourcecred","github","ISSUE","sourcecred","example-github","10","6","sourcecred","github","ISSUE","sourcecred","example-github","10"],"dstIndex":26,"srcIndex":26},{"address":["sourcecred","github","REFERENCES","6","sourcecred","github","ISSUE","sourcecred","example-github","10","6","sourcecred","github","ISSUE","sourcecred","example-github","2"],"dstIndex":30,"srcIndex":26},{"address":["sourcecred","github","REFERENCES","6","sourcecred","github","ISSUE","sourcecred","example-github","12","4","sourcecred","git","COMMIT","ec91adb718a6045b492303f00d8e8beb957dc780"],"dstIndex":4,"srcIndex":28},{"address":["sourcecred","github","REFERENCES","6","sourcecred","github","ISSUE","sourcecred","example-github","12","4","sourcecred","git","COMMIT","ecc889dc94cf6da17ae6eab5bb7b7155f577519d"],"dstIndex":5,"srcIndex":28},{"address":["sourcecred","github","REFERENCES","6","sourcecred","github","ISSUE","sourcecred","example-github","2","6","sourcecred","github","ISSUE","sourcecred","example-github","1"],"dstIndex":25,"srcIndex":30},{"address":["sourcecred","github","REFERENCES","6","sourcecred","github","PULL","sourcecred","example-github","5","5","sourcecred","github","USERLIKE","USER","wchargin"],"dstIndex":43,"srcIndex":36},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","373768703","6","sourcecred","github","ISSUE","sourcecred","example-github","6"],"dstIndex":32,"srcIndex":10},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","373768850","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","373768538"],"dstIndex":19,"srcIndex":11},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576185","6","sourcecred","github","PULL","sourcecred","example-github","5"],"dstIndex":36,"srcIndex":12},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576220","7","sourcecred","github","REVIEW","sourcecred","example-github","5","100313899"],"dstIndex":39,"srcIndex":13},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576248","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":14},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576273","5","sourcecred","github","USERLIKE","USER","wchargin"],"dstIndex":43,"srcIndex":15},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576920","6","sourcecred","github","ISSUE","sourcecred","example-github","1"],"dstIndex":25,"srcIndex":16},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576920","6","sourcecred","github","ISSUE","sourcecred","example-github","2"],"dstIndex":30,"srcIndex":16},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576920","6","sourcecred","github","PULL","sourcecred","example-github","3"],"dstIndex":35,"srcIndex":16},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576920","7","sourcecred","github","REVIEW","sourcecred","example-github","5","100313899"],"dstIndex":39,"srcIndex":16},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576920","9","sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],"dstIndex":24,"srcIndex":16},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","385223316","6","sourcecred","github","ISSUE","sourcecred","example-github","2"],"dstIndex":30,"srcIndex":20},{"address":["sourcecred","github","REFERENCES","8","sourcecred","github","COMMENT","PULL","sourcecred","example-github","3","369162222","6","sourcecred","github","ISSUE","sourcecred","example-github","2"],"dstIndex":30,"srcIndex":22}],"nodes":[["sourcecred","git","COMMIT","0a223346b4e6dec0127b1e6aa892c4ee0424b66a"],["sourcecred","git","COMMIT","6bd1b4c0b719c22c688a74863be07a699b7b9b34"],["sourcecred","git","COMMIT","6d5b3aa31ebb68a06ceb46bbd6cf49b6ccd6f5e6"],["sourcecred","git","COMMIT","c430bd74455105f77215ece51945094ceeee6c86"],["sourcecred","git","COMMIT","ec91adb718a6045b492303f00d8e8beb957dc780"],["sourcecred","git","COMMIT","ecc889dc94cf6da17ae6eab5bb7b7155f577519d"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420811872"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813013"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813206"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","11","420813621"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","373768703"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","373768850"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576185"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576220"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576248"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576273"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576920"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","2","385576936"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","373768442"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","373768538"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","385223316"],["sourcecred","github","COMMENT","ISSUE","sourcecred","example-github","6","417104047"],["sourcecred","github","COMMENT","PULL","sourcecred","example-github","3","369162222"],["sourcecred","github","COMMENT","PULL","sourcecred","example-github","5","396430464"],["sourcecred","github","COMMENT","REVIEW","sourcecred","example-github","5","100313899","171460198"],["sourcecred","github","ISSUE","sourcecred","example-github","1"],["sourcecred","github","ISSUE","sourcecred","example-github","10"],["sourcecred","github","ISSUE","sourcecred","example-github","11"],["sourcecred","github","ISSUE","sourcecred","example-github","12"],["sourcecred","github","ISSUE","sourcecred","example-github","13"],["sourcecred","github","ISSUE","sourcecred","example-github","2"],["sourcecred","github","ISSUE","sourcecred","example-github","4"],["sourcecred","github","ISSUE","sourcecred","example-github","6"],["sourcecred","github","ISSUE","sourcecred","example-github","7"],["sourcecred","github","ISSUE","sourcecred","example-github","8"],["sourcecred","github","PULL","sourcecred","example-github","3"],["sourcecred","github","PULL","sourcecred","example-github","5"],["sourcecred","github","PULL","sourcecred","example-github","9"],["sourcecred","github","REPO","sourcecred","example-github"],["sourcecred","github","REVIEW","sourcecred","example-github","5","100313899"],["sourcecred","github","REVIEW","sourcecred","example-github","5","100314038"],["sourcecred","github","USERLIKE","BOT","credbot"],["sourcecred","github","USERLIKE","USER","decentralion"],["sourcecred","github","USERLIKE","USER","wchargin"]]}],"scores":[0.02176357629452466,0.0011982455096751293,0.014130380391244092,0.0031449444172878174,0.01363997885528687,0.01363997885528687,0.0013228833532857442,0.0034622695982772875,0.006266345142945584,0.0018169268120504086,0.0012920287681048876,0.0012920933566244767,0.001291954952653929,0.0012920933566244767,0.0012920933566244767,0.0012920933566244767,0.001291869780979746,0.0012922010041571254,0.0013835638323824034,0.0017066140778612396,0.001383379418706535,0.0009754593334172495,0.0024170418884334847,0.01542041419510214,0.04064015162449017,0.027878480583040766,0.027024949212945043,0.031738615236138926,0.01931855156709714,0.03315781115847288,0.02871935025854917,0.017343476784993136,0.02133963520469102,0.017343476784993136,0.017343476784993136,0.04807261129492345,0.07912329807508121,0.055103153601238894,0.2616617802304554,0.02513632527649076,0.004400181623206724,0.0002582389976061568,0.07263358456184818,0.05775442123058088],"syntheticLoopWeight":0.001,"toWeights":[2,2,2,2,2,2,1,0.5,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,1,1,2,0.5,0.5,0.5,0.5,0.5,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,4,4,1,1,1,1,4,4,8,2,2,2,8,4,4,4,2,2,2,4,1,1,1,4,1,4,1,2,2,2,2,2,1,2,1,4,1,1,1,2,2,4,1,1,2,2]}] \ No newline at end of file diff --git a/src/cli/load.js b/src/cli/load.js index 99bef95..0cdfc52 100644 --- a/src/cli/load.js +++ b/src/cli/load.js @@ -165,31 +165,58 @@ export function makeLoadCommand( } export const loadDefaultPlugins = async (options: LoadOptions) => { + const sourcecredCommand = (args) => [ + process.execPath, + "--max_old_space_size=8192", + process.argv[1], + ...args, + ]; const tasks = [ ...Common.defaultPlugins().map((pluginName) => ({ id: `load-${pluginName}`, - cmd: [ - process.execPath, - "--max_old_space_size=8192", - process.argv[1], + cmd: sourcecredCommand([ "load", ...options.repoIds.map((repoId) => repoIdToString(repoId)), "--output", repoIdToString(options.output), "--plugin", pluginName, - ], + ]), deps: [], })), ]; - const {success} = await execDependencyGraph(tasks, {taskPassLabel: "DONE"}); - if (success) { - addToRepoIdRegistry(options.output); - return; - } else { + const {success: loadSuccess} = await execDependencyGraph(tasks, { + taskPassLabel: "DONE", + }); + if (!loadSuccess) { throw new Error("Load tasks failed."); } + addToRepoIdRegistry(options.output); + // HACK: Logically, we should have the PagerankTask be included in the + // first execDependencyGraph run, depending on the other tasks completing. + // + // However, running pagerank depends on loading the graph + // (analysis/loadGraph), which depends on the relevant repo being present + // in the RepoIdRegistry. And it is only in the RepoIdRegistry after the + // call to execDependencyGraph has been successful. + // + // As a simple hack, we just call execDependencyGraph again with the + // pagerank command after the first one has been successful. This does have + // the awkward effect that CLI users will see two blocks of "task: SUCCESS" + // information from execDependencyGraph. + const pagerankTask = { + id: "run-pagerank", + cmd: sourcecredCommand(["pagerank", repoIdToString(options.output)]), + deps: [], + }; + const {success: pagerankSuccess} = await execDependencyGraph([pagerankTask], { + taskPassLabel: "DONE", + }); + if (!pagerankSuccess) { + throw new Error("Pagerank task failed."); + } + return; }; export const loadIndividualPlugin = async ( diff --git a/src/cli/load.test.js b/src/cli/load.test.js index 216c1ee..e573a9b 100644 --- a/src/cli/load.test.js +++ b/src/cli/load.test.js @@ -464,18 +464,18 @@ describe("cli/load", () => { const fooBaz = makeRepoId("foo", "baz"); it("creates a load sub-task per plugin", async () => { - execDependencyGraph.mockResolvedValueOnce({success: true}); + execDependencyGraph.mockResolvedValue({success: true}); await loadDefaultPlugins({ output: fooCombined, repoIds: [fooBar, fooBaz], }); - expect(execDependencyGraph).toHaveBeenCalledTimes(1); - const tasks = execDependencyGraph.mock.calls[0][0]; - expect(tasks).toHaveLength(["git", "github"].length); - expect(tasks.map((task) => task.id)).toEqual( + expect(execDependencyGraph).toHaveBeenCalledTimes(2); + const loadTasks = execDependencyGraph.mock.calls[0][0]; + expect(loadTasks).toHaveLength(["git", "github"].length); + expect(loadTasks.map((task) => task.id)).toEqual( expect.arrayContaining(["load-git", "load-github"]) ); - for (const task of tasks) { + for (const task of loadTasks) { expect(task.cmd).toEqual([ expect.stringMatching(/\bnode\b/), expect.stringMatching(/--max_old_space_size=/), @@ -491,12 +491,35 @@ describe("cli/load", () => { } }); + it("creates a pagerank task after load is successful", async () => { + execDependencyGraph.mockResolvedValue({success: true}); + await loadDefaultPlugins({ + output: fooCombined, + repoIds: [fooBar, fooBaz], + }); + expect(execDependencyGraph).toHaveBeenCalledTimes(2); + const pagerankTasks = execDependencyGraph.mock.calls[1][0]; + expect(pagerankTasks).toHaveLength(1); + const task = pagerankTasks[0]; + expect(task).toEqual({ + id: "run-pagerank", + deps: [], + cmd: [ + expect.stringMatching(/\bnode\b/), + expect.stringMatching(/--max_old_space_size=/), + process.argv[1], + "pagerank", + "foo/combined", + ], + }); + }); + it("updates RepoIdRegistry on success", async () => { const directory = newSourcecredDirectory(); expect(RepoIdRegistry.getRegistry(directory)).toEqual( RepoIdRegistry.emptyRegistry() ); - execDependencyGraph.mockResolvedValueOnce({success: true}); + execDependencyGraph.mockResolvedValue({success: true}); await loadDefaultPlugins({ output: fooCombined, repoIds: [fooBar, fooBaz], @@ -510,7 +533,7 @@ describe("cli/load", () => { expect(RepoIdRegistry.getRegistry(directory)).toEqual(expectedRegistry); }); - it("throws an error on execDependencyGraph failure", async () => { + it("throws an load error on first execDependencyGraph failure", async () => { execDependencyGraph.mockResolvedValueOnce({success: false}); const result = loadDefaultPlugins({ output: fooCombined, @@ -519,5 +542,16 @@ describe("cli/load", () => { expect(result).rejects.toThrow("Load tasks failed."); }); + + it("throws an pagerank error on second execDependencyGraph failure", async () => { + execDependencyGraph.mockResolvedValueOnce({success: true}); + execDependencyGraph.mockResolvedValueOnce({success: false}); + const result = loadDefaultPlugins({ + output: fooCombined, + repoIds: [fooBar, fooBaz], + }); + + expect(result).rejects.toThrow("Pagerank task failed."); + }); }); });