add `api/load` (#1251)

This adds a new module, `api/load`, which implements the logic that will
underly the new `sourcecred load` command. The `api` package is a new
folder that will contain the logic that powers the CLI (but will be
callable directly as we improve SourceCred). As a heuristic, nontrivial
logic in `cli/` should be factored out to `api/`.

In the future, we will likely want to refactor these APIs to
make them more atomic/composable. `api/load` does "all the things" in
terms of loading data, computing cred, and writing it to disk. I'm going
with the simplest approach here (mirroring existing functionality) so
that we can merge #1233 and realize its many benefits more easily.

This work is factored out of #1233. Thanks to @Beanow for [review]
of the module, which resulted in several changes (e.g. organizing it
under api/, having the TaskReporter be dependency injected).

[review]: https://github.com/sourcecred/sourcecred/pull/1233#pullrequestreview-263633643

Test plan: `api/load` is tested (via mocking unit tests). Run `yarn test`

src/api/load.js

@@ -0,0 +1,71 @@
+// @flow
+
+import fs from "fs-extra";
+import path from "path";
+
+import {TaskReporter} from "../util/taskReporter";
+import {loadGraph} from "../plugins/github/loadGraph";
+import {
+  type TimelineCredParameters,
+  TimelineCred,
+} from "../analysis/timeline/timelineCred";
+
+import {DEFAULT_CRED_CONFIG} from "../plugins/defaultCredConfig";
+
+import {type Project} from "../core/project";
+import {setupProjectDirectory} from "../core/project_io";
+
+export type LoadOptions = {|
+  +project: Project,
+  +params: TimelineCredParameters,
+  +sourcecredDirectory: string,
+  +githubToken: string,
+|};
+
+/**
+ * Loads and computes cred for a project.
+ *
+ * Loads the combined Graph for the specified project, saves it to disk,
+ * and computes cred for it using the provided TimelineCredParameters.
+ *
+ * A project directory will be created for the given project within the
+ * provided sourcecredDirectory, using the APIs in core/project_io. Within this
+ * project directory, there will be a `cred.json` file containing filtered
+ * timeline cred, and a `graph.json` file containing the combined graph.
+ *
+ * In the future, we should consider splitting this into cleaner, more atomic
+ * APIs (e.g. one for loading the graph; another for computing cred).
+ */
+export async function load(
+  options: LoadOptions,
+  taskReporter: TaskReporter
+): Promise<void> {
+  const {project, params, sourcecredDirectory, githubToken} = options;
+  const loadTask = `load-${options.project.id}`;
+  taskReporter.start(loadTask);
+  const cacheDirectory = path.join(sourcecredDirectory, "cache");
+  await fs.mkdirp(cacheDirectory);
+
+  // future: support loading more plugins, and merging their graphs
+  const githubOptions = {
+    repoIds: project.repoIds,
+    token: githubToken,
+    cacheDirectory,
+  };
+  const graph = await loadGraph(githubOptions, taskReporter);
+
+  const projectDirectory = await setupProjectDirectory(
+    project,
+    sourcecredDirectory
+  );
+  const graphFile = path.join(projectDirectory, "graph.json");
+  await fs.writeFile(graphFile, JSON.stringify(graph.toJSON()));
+
+  taskReporter.start("compute-cred");
+  const cred = await TimelineCred.compute(graph, params, DEFAULT_CRED_CONFIG);
+  const credJSON = cred.toJSON();
+  const credFile = path.join(projectDirectory, "cred.json");
+  await fs.writeFile(credFile, JSON.stringify(credJSON));
+  taskReporter.finish("compute-cred");
+  taskReporter.finish(loadTask);
+}

src/api/load.test.js

@@ -0,0 +1,134 @@
+// @flow
+
+import deepFreeze from "deep-freeze";
+import tmp from "tmp";
+import path from "path";
+import fs from "fs-extra";
+
+import type {Options as LoadGraphOptions} from "../plugins/github/loadGraph";
+import type {Project} from "../core/project";
+import {
+  directoryForProjectId,
+  getProjectIds,
+  loadProject,
+} from "../core/project_io";
+import {makeRepoId} from "../core/repoId";
+import {defaultWeights} from "../analysis/weights";
+import {NodeAddress} from "../core/graph";
+import {TestTaskReporter} from "../util/taskReporter";
+import {load, type LoadOptions} from "./load";
+import {DEFAULT_CRED_CONFIG} from "../plugins/defaultCredConfig";
+
+type JestMockFn = $Call<typeof jest.fn>;
+jest.mock("../plugins/github/loadGraph", () => ({
+  loadGraph: jest.fn(),
+}));
+const loadGraph: JestMockFn = (require("../plugins/github/loadGraph")
+  .loadGraph: any);
+
+jest.mock("../analysis/timeline/timelineCred", () => ({
+  TimelineCred: {compute: jest.fn()},
+}));
+const timelineCredCompute: JestMockFn = (require("../analysis/timeline/timelineCred")
+  .TimelineCred.compute: any);
+
+describe("api/load", () => {
+  const fakeTimelineCred = deepFreeze({
+    toJSON: () => ({is: "fake-timeline-cred"}),
+  });
+  const fakeGraph = deepFreeze({toJSON: () => ({is: "fake-graph"})});
+  beforeEach(() => {
+    jest.clearAllMocks();
+    loadGraph.mockResolvedValue(fakeGraph);
+    timelineCredCompute.mockResolvedValue(fakeTimelineCred);
+  });
+  const project: Project = deepFreeze({
+    id: "foo",
+    repoIds: [makeRepoId("foo", "bar")],
+  });
+  const githubToken = "EXAMPLE_TOKEN";
+  const weights = defaultWeights();
+  // Tweaks the weights so that we can ensure we aren't overriding with default weights
+  weights.nodeManualWeights.set(NodeAddress.empty, 33);
+  // Deep freeze will freeze the weights, too
+  const params = deepFreeze({alpha: 0.05, intervalDecay: 0.5, weights});
+  const example = () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    const taskReporter = new TestTaskReporter();
+    const options: LoadOptions = {
+      sourcecredDirectory,
+      githubToken,
+      params,
+      project,
+    };
+    return {options, taskReporter, sourcecredDirectory};
+  };
+
+  it("sets up a project directory for the project", async () => {
+    const {options, taskReporter, sourcecredDirectory} = example();
+    await load(options, taskReporter);
+    expect(await getProjectIds(sourcecredDirectory)).toEqual([project.id]);
+    expect(await loadProject(project.id, sourcecredDirectory)).toEqual(project);
+  });
+
+  it("calls github loadGraph with the right options", async () => {
+    const {options, taskReporter, sourcecredDirectory} = example();
+    await load(options, taskReporter);
+    const cacheDirectory = path.join(sourcecredDirectory, "cache");
+    const expectedLoadGraphOptions: LoadGraphOptions = {
+      repoIds: project.repoIds,
+      token: githubToken,
+      cacheDirectory,
+    };
+    expect(loadGraph).toHaveBeenCalledWith(
+      expectedLoadGraphOptions,
+      taskReporter
+    );
+  });
+
+  it("saves the resultant graph to disk", async () => {
+    const {options, taskReporter, sourcecredDirectory} = example();
+    await load(options, taskReporter);
+    const projectDirectory = directoryForProjectId(
+      project.id,
+      sourcecredDirectory
+    );
+    const graphFile = path.join(projectDirectory, "graph.json");
+    const graphJSON = JSON.parse(await fs.readFile(graphFile));
+    expect(graphJSON).toEqual(fakeGraph.toJSON());
+  });
+
+  it("calls TimelineCred.compute with the right graph and options", async () => {
+    const {options, taskReporter} = example();
+    await load(options, taskReporter);
+    expect(timelineCredCompute).toHaveBeenCalledWith(
+      fakeGraph,
+      params,
+      DEFAULT_CRED_CONFIG
+    );
+  });
+
+  it("saves the resultant cred.json to disk", async () => {
+    const {options, taskReporter, sourcecredDirectory} = example();
+    await load(options, taskReporter);
+    const projectDirectory = directoryForProjectId(
+      project.id,
+      sourcecredDirectory
+    );
+    const credFile = path.join(projectDirectory, "cred.json");
+    const credJSON = JSON.parse(await fs.readFile(credFile));
+    expect(credJSON).toEqual(fakeTimelineCred.toJSON());
+  });
+
+  it("gives the right tasks to the TaskReporter", async () => {
+    const {options, taskReporter} = example();
+    await load(options, taskReporter);
+    expect(taskReporter.activeTasks()).toEqual([]);
+    expect(taskReporter.entries()).toEqual([
+      {type: "START", taskId: "load-foo"},
+      {type: "START", taskId: "compute-cred"},
+      {type: "FINISH", taskId: "compute-cred"},
+      {type: "FINISH", taskId: "load-foo"},
+    ]);
+  });
+});