From a46500d70452e4d9379f6f21f251173b3e91b7f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Wed, 10 Jul 2019 01:06:26 +0100 Subject: [PATCH] Modify `sourcecred load` to save timeline cred Test plan: Observe changes to the snapshot for example-github-load. `yarn test --full` passes. --- .../data/sourcecred/example-github/cred.json | 1 + src/cli/load.js | 49 +++++++++++++++++-- src/cli/load.test.js | 20 +++++++- 3 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 sharness/__snapshots__/example-github-load/data/sourcecred/example-github/cred.json diff --git a/sharness/__snapshots__/example-github-load/data/sourcecred/example-github/cred.json b/sharness/__snapshots__/example-github-load/data/sourcecred/example-github/cred.json new file mode 100644 index 0000000..bde7e92 --- /dev/null +++ b/sharness/__snapshots__/example-github-load/data/sourcecred/example-github/cred.json @@ -0,0 +1 @@ +{"intervals":[{"startTimeMs":1519516800000,"endTimeMs":1520121600000},{"startTimeMs":1520121600000,"endTimeMs":1520726400000},{"startTimeMs":1520726400000,"endTimeMs":1521331200000},{"startTimeMs":1521331200000,"endTimeMs":1521936000000},{"startTimeMs":1521936000000,"endTimeMs":1522540800000},{"startTimeMs":1522540800000,"endTimeMs":1523145600000},{"startTimeMs":1523145600000,"endTimeMs":1523750400000},{"startTimeMs":1523750400000,"endTimeMs":1524355200000},{"startTimeMs":1524355200000,"endTimeMs":1524960000000},{"startTimeMs":1524960000000,"endTimeMs":1525564800000},{"startTimeMs":1525564800000,"endTimeMs":1526169600000},{"startTimeMs":1526169600000,"endTimeMs":1526774400000},{"startTimeMs":1526774400000,"endTimeMs":1527379200000},{"startTimeMs":1527379200000,"endTimeMs":1527984000000},{"startTimeMs":1527984000000,"endTimeMs":1528588800000},{"startTimeMs":1528588800000,"endTimeMs":1529193600000},{"startTimeMs":1529193600000,"endTimeMs":1529798400000},{"startTimeMs":1529798400000,"endTimeMs":1530403200000},{"startTimeMs":1530403200000,"endTimeMs":1531008000000},{"startTimeMs":1531008000000,"endTimeMs":1531612800000},{"startTimeMs":1531612800000,"endTimeMs":1532217600000},{"startTimeMs":1532217600000,"endTimeMs":1532822400000},{"startTimeMs":1532822400000,"endTimeMs":1533427200000},{"startTimeMs":1533427200000,"endTimeMs":1534032000000},{"startTimeMs":1534032000000,"endTimeMs":1534636800000},{"startTimeMs":1534636800000,"endTimeMs":1535241600000},{"startTimeMs":1535241600000,"endTimeMs":1535846400000},{"startTimeMs":1535846400000,"endTimeMs":1536451200000},{"startTimeMs":1536451200000,"endTimeMs":1537056000000},{"startTimeMs":1537056000000,"endTimeMs":1537660800000},{"startTimeMs":1537660800000,"endTimeMs":1538265600000},{"startTimeMs":1538265600000,"endTimeMs":1538870400000},{"startTimeMs":1538870400000,"endTimeMs":1539475200000},{"startTimeMs":1539475200000,"endTimeMs":1540080000000},{"startTimeMs":1540080000000,"endTimeMs":1540684800000},{"startTimeMs":1540684800000,"endTimeMs":1541289600000},{"startTimeMs":1541289600000,"endTimeMs":1541894400000},{"startTimeMs":1541894400000,"endTimeMs":1542499200000},{"startTimeMs":1542499200000,"endTimeMs":1543104000000},{"startTimeMs":1543104000000,"endTimeMs":1543708800000},{"startTimeMs":1543708800000,"endTimeMs":1544313600000},{"startTimeMs":1544313600000,"endTimeMs":1544918400000},{"startTimeMs":1544918400000,"endTimeMs":1545523200000},{"startTimeMs":1545523200000,"endTimeMs":1546128000000},{"startTimeMs":1546128000000,"endTimeMs":1546732800000},{"startTimeMs":1546732800000,"endTimeMs":1547337600000},{"startTimeMs":1547337600000,"endTimeMs":1547942400000},{"startTimeMs":1547942400000,"endTimeMs":1548547200000}],"addressToCred":{"N\u0000sourcecred\u0000github\u0000REPO\u0000sourcecred\u0000example-github\u0000":[6.434277085743278,3.2187001409961105,2.9537008390444695,3.3669202249563632,1.6844790015992535,0.8432534368356913,0.4226399752299933,0.21233166427887087,0.1071748798625617,1.6535003231148968,0.8279279390642292,0.4151594830901334,0.20880775901265353,0.10568567754170251,0.05419793288678042,0.01107556028113049,0.006325160932577391,0.391887174134945,0.19781636029641406,0.10021435910895005,0.05121169842562323,0.026593742059359833,0.014158406027485857,0.007782237567626448,0.004416704702613905,0.0025597904160237863,0.3309468491022413,0.1833239543651179,3.8662293710831137,1.9280267792288135,0.9683855479057963,0.48750710772208333,0.24651468659925346,0.12576586231522624,0.06528611135866129,0.034987152889186975,0.019769892758210943,0.012046681939860636,0.007981247381836032,0.00560700369251145,0.00392227632622292,0.002514595555540504,0.00137796216092967,0.0006307162324228583,0.0002489262811051821,0.00009041906147473655,0.000032443854636261745,0.000019245966928020832],"N\u0000sourcecred\u0000github\u0000USERLIKE\u0000USER\u0000decentralion\u0000":[9.383869639663432,4.691404861944296,5.407033783069464,4.710039491158759,2.3542990795642016,1.1764257521110257,0.5874968415385614,0.2930472936432958,0.14585208714512637,3.367558953101356,1.6854511612955203,0.8443081404654077,0.4235721757793936,0.21292350141151434,0.10718062099935051,0.024900766153114956,0.01234421271106403,0.5639545904301708,0.2826874903289227,0.14116839002688855,0.07018698540293254,0.03469827124411613,0.017043428850844578,0.008340605522915224,0.004101657419219121,0.0020470494713788713,0.2330964724510903,0.11911297405802071,1.9437472461067256,1.2458984494463696,0.6241050193500056,0.3126272039780528,0.1565536991433818,0.07836083092825474,0.039204376624594785,0.019603716512520068,0.009792000539440526,0.004877883689977619,0.002415531283206432,0.0011833764411626655,0.0005707084414700211,0.00027081941634511306,0.00012783639288752084,0.00006135289874258643,0.0000305382491069489,0.000015865125198178594,0.00000853390543554515,8.18684979546365e-7],"N\u0000sourcecred\u0000github\u0000USERLIKE\u0000USER\u0000wchargin\u0000":[3.6161303603365673,1.808595138055704,0.8429662169305362,0.41496050884124025,0.2082009204357985,0.10482424788897443,0.05312815846143868,0.02726520635670421,0.014304162854873638,2.2125191718986437,1.10458790120448,0.5507113907845924,0.2739375898456065,0.13583138140098566,0.0671968204068995,0.5622879545500101,0.2812501476404985,0.5828425897456103,0.2907110997589679,0.14553090501705676,0.07316266211904013,0.03697655251687019,0.018793983029648593,0.00957810041733136,0.004857695550904171,0.0024326270136827742,0.2691433657914406,0.1320069450632447,4.1818127134539065,1.8168815303339467,0.9072849705401523,0.45306779096702615,0.22629379832915772,0.11306291780801501,0.0565074977435401,0.02825222067154737,0.014135968052593194,0.00708610060603924,0.0035664608648019987,0.00180761963284155,0.0009247895955320864,0.00047692960215594074,0.0002460381163630061,0.000125584355882677,0.00006293037820568283,0.00003086918845813727,0.00001483325139261278,0.000010864893434532599]}} \ No newline at end of file diff --git a/src/cli/load.js b/src/cli/load.js index 29e44c6..4e67cad 100644 --- a/src/cli/load.js +++ b/src/cli/load.js @@ -3,6 +3,7 @@ import mkdirp from "mkdirp"; import path from "path"; +import chalk from "chalk"; import fs from "fs-extra"; import stringify from "json-stable-stringify"; @@ -11,13 +12,17 @@ import {repoIdToString, stringToRepoId, type RepoId} from "../core/repoId"; import dedent from "../util/dedent"; import type {Command} from "./command"; import * as Common from "./common"; +import {Graph} from "../core/graph"; import {loadGraph, type LoadGraphResult} from "../analysis/loadGraph"; import {type IBackendAdapterLoader} from "../analysis/analysisAdapter"; +import {TimelineCred} from "../analysis/timeline/timelineCred"; +import {defaultWeights} from "../analysis/weights"; import execDependencyGraph from "../tools/execDependencyGraph"; import {loadGithubData} from "../plugins/github/loadGithubData"; import {loadGitData} from "../plugins/git/loadGitData"; import {defaultAdapterLoaders} from "./pagerank"; +import {DEFAULT_CRED_CONFIG} from "../plugins/defaultCredConfig"; function usage(print: (string) => void): void { print( @@ -170,7 +175,9 @@ export function makeLoadCommand( export type SaveGraph = ( $ReadOnlyArray, RepoId -) => Promise; +) => Promise; + +export type SaveCred = (Graph, RepoId) => Promise; /** * A wrapper around the default plugin loader. @@ -179,7 +186,10 @@ export type SaveGraph = ( * (saveGraph depends on the directory state that's generated by running * load.) */ -export const makeLoadDefaultPlugins = (saveGraph: SaveGraph) => { +export const makeLoadDefaultPlugins = ( + saveGraph: SaveGraph, + saveCred: SaveCred +) => { return async (options: LoadOptions) => { const sourcecredCommand = (args) => [ process.execPath, @@ -209,11 +219,39 @@ export const makeLoadDefaultPlugins = (saveGraph: SaveGraph) => { throw new Error("Load tasks failed."); } addToRepoIdRegistry(options.output); - await saveGraph(defaultAdapterLoaders(), options.output); + const graph = await saveGraph(defaultAdapterLoaders(), options.output); + await saveCred(graph, options.output); return; }; }; +export const saveCred = async (graph: Graph, repoId: RepoId) => { + const id = "compute-cred"; + console.time(id); + // Imitate the task labels from execDependencyGraph for visual consistency + console.log(chalk.bgBlue.bold.white(" GO ") + ` ${id}`); + const cred = await TimelineCred.compute( + graph, + { + alpha: 0.05, + intervalDecay: 0.5, + weights: defaultWeights(), + }, + DEFAULT_CRED_CONFIG + ); + const credFile = path.join( + Common.sourcecredDirectory(), + "data", + repoIdToString(repoId), + "cred.json" + ); + const credJSON = cred.toJSON(); + await fs.writeFile(credFile, JSON.stringify(credJSON)); + const badge = chalk.bgGreen.bold.white(" DONE "); + console.log(`${badge} ${id}`); + console.timeEnd(id); +}; + export const loadIndividualPlugin = async ( plugin: Common.PluginName, options: LoadOptions @@ -266,7 +304,7 @@ function addToRepoIdRegistry(repoId) { export async function saveGraph( adapterLoaders: $ReadOnlyArray, repoId: RepoId -) { +): Promise { const loadGraphResult: LoadGraphResult = await loadGraph( Common.sourcecredDirectory(), adapterLoaders, @@ -284,6 +322,7 @@ export async function saveGraph( ); const graphJSON = graph.toJSON(); await fs.writeFile(graphFile, stringify(graphJSON)); + return graph; } export const help: Command = async (args, std) => { @@ -296,7 +335,7 @@ export const help: Command = async (args, std) => { } }; -const loadDefaultPlugins = makeLoadDefaultPlugins(saveGraph); +const loadDefaultPlugins = makeLoadDefaultPlugins(saveGraph, saveCred); const load = makeLoadCommand(loadIndividualPlugin, loadDefaultPlugins); export default load; diff --git a/src/cli/load.test.js b/src/cli/load.test.js index 1397711..b4af98a 100644 --- a/src/cli/load.test.js +++ b/src/cli/load.test.js @@ -3,6 +3,7 @@ import path from "path"; import tmp from "tmp"; +import {Graph} from "../core/graph"; import {run} from "./testUtil"; import {defaultPlugins} from "./common"; import { @@ -467,7 +468,7 @@ describe("cli/load", () => { const fooCombined = makeRepoId("foo", "combined"); const fooBar = makeRepoId("foo", "bar"); const fooBaz = makeRepoId("foo", "baz"); - const loadDefaultPlugins = makeLoadDefaultPlugins(jest.fn()); + const loadDefaultPlugins = makeLoadDefaultPlugins(jest.fn(), jest.fn()); it("creates a load sub-task per plugin", async () => { execDependencyGraph.mockResolvedValue({success: true}); @@ -518,7 +519,7 @@ describe("cli/load", () => { it("calls saveGraph on success", async () => { const saveGraph = jest.fn(); - const loadDefaultPlugins = makeLoadDefaultPlugins(saveGraph); + const loadDefaultPlugins = makeLoadDefaultPlugins(saveGraph, jest.fn()); execDependencyGraph.mockResolvedValue({success: true}); await loadDefaultPlugins({ output: fooCombined, @@ -531,6 +532,21 @@ describe("cli/load", () => { ); }); + it("calls saveCred with the output of saveGraph", async () => { + // $ExpectFlowError + const graph: Graph = "pretend_graph"; + const saveGraph = jest.fn().mockResolvedValue(graph); + const saveCred = jest.fn(); + const loadDefaultPlugins = makeLoadDefaultPlugins(saveGraph, saveCred); + execDependencyGraph.mockResolvedValue({success: true}); + await loadDefaultPlugins({ + output: fooCombined, + repoIds: [fooBar, fooBaz], + }); + expect(saveCred).toHaveBeenCalledTimes(1); + expect(saveCred).toHaveBeenCalledWith(graph, fooCombined); + }); + it("throws an load error on first execDependencyGraph failure", async () => { execDependencyGraph.mockResolvedValueOnce({success: false}); const result = loadDefaultPlugins({