From d0660c9366bdff2db8568b38b05929de52f9b665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Mon, 22 Jul 2019 15:41:24 +0100 Subject: [PATCH] add `api/load` (#1251) This adds a new module, `api/load`, which implements the logic that will underly the new `sourcecred load` command. The `api` package is a new folder that will contain the logic that powers the CLI (but will be callable directly as we improve SourceCred). As a heuristic, nontrivial logic in `cli/` should be factored out to `api/`. In the future, we will likely want to refactor these APIs to make them more atomic/composable. `api/load` does "all the things" in terms of loading data, computing cred, and writing it to disk. I'm going with the simplest approach here (mirroring existing functionality) so that we can merge #1233 and realize its many benefits more easily. This work is factored out of #1233. Thanks to @Beanow for [review] of the module, which resulted in several changes (e.g. organizing it under api/, having the TaskReporter be dependency injected). [review]: https://github.com/sourcecred/sourcecred/pull/1233#pullrequestreview-263633643 Test plan: `api/load` is tested (via mocking unit tests). Run `yarn test` --- src/api/load.js | 71 +++++++++++++++++++++++ src/api/load.test.js | 134 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 src/api/load.js create mode 100644 src/api/load.test.js diff --git a/src/api/load.js b/src/api/load.js new file mode 100644 index 0000000..f212c90 --- /dev/null +++ b/src/api/load.js @@ -0,0 +1,71 @@ +// @flow + +import fs from "fs-extra"; +import path from "path"; + +import {TaskReporter} from "../util/taskReporter"; +import {loadGraph} from "../plugins/github/loadGraph"; +import { + type TimelineCredParameters, + TimelineCred, +} from "../analysis/timeline/timelineCred"; + +import {DEFAULT_CRED_CONFIG} from "../plugins/defaultCredConfig"; + +import {type Project} from "../core/project"; +import {setupProjectDirectory} from "../core/project_io"; + +export type LoadOptions = {| + +project: Project, + +params: TimelineCredParameters, + +sourcecredDirectory: string, + +githubToken: string, +|}; + +/** + * Loads and computes cred for a project. + * + * Loads the combined Graph for the specified project, saves it to disk, + * and computes cred for it using the provided TimelineCredParameters. + * + * A project directory will be created for the given project within the + * provided sourcecredDirectory, using the APIs in core/project_io. Within this + * project directory, there will be a `cred.json` file containing filtered + * timeline cred, and a `graph.json` file containing the combined graph. + * + * In the future, we should consider splitting this into cleaner, more atomic + * APIs (e.g. one for loading the graph; another for computing cred). + */ +export async function load( + options: LoadOptions, + taskReporter: TaskReporter +): Promise { + const {project, params, sourcecredDirectory, githubToken} = options; + const loadTask = `load-${options.project.id}`; + taskReporter.start(loadTask); + const cacheDirectory = path.join(sourcecredDirectory, "cache"); + await fs.mkdirp(cacheDirectory); + + // future: support loading more plugins, and merging their graphs + const githubOptions = { + repoIds: project.repoIds, + token: githubToken, + cacheDirectory, + }; + const graph = await loadGraph(githubOptions, taskReporter); + + const projectDirectory = await setupProjectDirectory( + project, + sourcecredDirectory + ); + const graphFile = path.join(projectDirectory, "graph.json"); + await fs.writeFile(graphFile, JSON.stringify(graph.toJSON())); + + taskReporter.start("compute-cred"); + const cred = await TimelineCred.compute(graph, params, DEFAULT_CRED_CONFIG); + const credJSON = cred.toJSON(); + const credFile = path.join(projectDirectory, "cred.json"); + await fs.writeFile(credFile, JSON.stringify(credJSON)); + taskReporter.finish("compute-cred"); + taskReporter.finish(loadTask); +} diff --git a/src/api/load.test.js b/src/api/load.test.js new file mode 100644 index 0000000..b5ba502 --- /dev/null +++ b/src/api/load.test.js @@ -0,0 +1,134 @@ +// @flow + +import deepFreeze from "deep-freeze"; +import tmp from "tmp"; +import path from "path"; +import fs from "fs-extra"; + +import type {Options as LoadGraphOptions} from "../plugins/github/loadGraph"; +import type {Project} from "../core/project"; +import { + directoryForProjectId, + getProjectIds, + loadProject, +} from "../core/project_io"; +import {makeRepoId} from "../core/repoId"; +import {defaultWeights} from "../analysis/weights"; +import {NodeAddress} from "../core/graph"; +import {TestTaskReporter} from "../util/taskReporter"; +import {load, type LoadOptions} from "./load"; +import {DEFAULT_CRED_CONFIG} from "../plugins/defaultCredConfig"; + +type JestMockFn = $Call; +jest.mock("../plugins/github/loadGraph", () => ({ + loadGraph: jest.fn(), +})); +const loadGraph: JestMockFn = (require("../plugins/github/loadGraph") + .loadGraph: any); + +jest.mock("../analysis/timeline/timelineCred", () => ({ + TimelineCred: {compute: jest.fn()}, +})); +const timelineCredCompute: JestMockFn = (require("../analysis/timeline/timelineCred") + .TimelineCred.compute: any); + +describe("api/load", () => { + const fakeTimelineCred = deepFreeze({ + toJSON: () => ({is: "fake-timeline-cred"}), + }); + const fakeGraph = deepFreeze({toJSON: () => ({is: "fake-graph"})}); + beforeEach(() => { + jest.clearAllMocks(); + loadGraph.mockResolvedValue(fakeGraph); + timelineCredCompute.mockResolvedValue(fakeTimelineCred); + }); + const project: Project = deepFreeze({ + id: "foo", + repoIds: [makeRepoId("foo", "bar")], + }); + const githubToken = "EXAMPLE_TOKEN"; + const weights = defaultWeights(); + // Tweaks the weights so that we can ensure we aren't overriding with default weights + weights.nodeManualWeights.set(NodeAddress.empty, 33); + // Deep freeze will freeze the weights, too + const params = deepFreeze({alpha: 0.05, intervalDecay: 0.5, weights}); + const example = () => { + const sourcecredDirectory = tmp.dirSync().name; + const taskReporter = new TestTaskReporter(); + const options: LoadOptions = { + sourcecredDirectory, + githubToken, + params, + project, + }; + return {options, taskReporter, sourcecredDirectory}; + }; + + it("sets up a project directory for the project", async () => { + const {options, taskReporter, sourcecredDirectory} = example(); + await load(options, taskReporter); + expect(await getProjectIds(sourcecredDirectory)).toEqual([project.id]); + expect(await loadProject(project.id, sourcecredDirectory)).toEqual(project); + }); + + it("calls github loadGraph with the right options", async () => { + const {options, taskReporter, sourcecredDirectory} = example(); + await load(options, taskReporter); + const cacheDirectory = path.join(sourcecredDirectory, "cache"); + const expectedLoadGraphOptions: LoadGraphOptions = { + repoIds: project.repoIds, + token: githubToken, + cacheDirectory, + }; + expect(loadGraph).toHaveBeenCalledWith( + expectedLoadGraphOptions, + taskReporter + ); + }); + + it("saves the resultant graph to disk", async () => { + const {options, taskReporter, sourcecredDirectory} = example(); + await load(options, taskReporter); + const projectDirectory = directoryForProjectId( + project.id, + sourcecredDirectory + ); + const graphFile = path.join(projectDirectory, "graph.json"); + const graphJSON = JSON.parse(await fs.readFile(graphFile)); + expect(graphJSON).toEqual(fakeGraph.toJSON()); + }); + + it("calls TimelineCred.compute with the right graph and options", async () => { + const {options, taskReporter} = example(); + await load(options, taskReporter); + expect(timelineCredCompute).toHaveBeenCalledWith( + fakeGraph, + params, + DEFAULT_CRED_CONFIG + ); + }); + + it("saves the resultant cred.json to disk", async () => { + const {options, taskReporter, sourcecredDirectory} = example(); + await load(options, taskReporter); + const projectDirectory = directoryForProjectId( + project.id, + sourcecredDirectory + ); + const credFile = path.join(projectDirectory, "cred.json"); + const credJSON = JSON.parse(await fs.readFile(credFile)); + expect(credJSON).toEqual(fakeTimelineCred.toJSON()); + }); + + it("gives the right tasks to the TaskReporter", async () => { + const {options, taskReporter} = example(); + await load(options, taskReporter); + expect(taskReporter.activeTasks()).toEqual([]); + expect(taskReporter.entries()).toEqual([ + {type: "START", taskId: "load-foo"}, + {type: "START", taskId: "compute-cred"}, + {type: "FINISH", taskId: "compute-cred"}, + {type: "FINISH", taskId: "load-foo"}, + ]); + }); +});