diff --git a/src/api/load.js b/src/api/load.js index 6f4af1a..81847e1 100644 --- a/src/api/load.js +++ b/src/api/load.js @@ -1,28 +1,13 @@ // @flow -import fs from "fs-extra"; -import path from "path"; -import stringify from "json-stable-stringify"; - -import {TaskReporter} from "../util/taskReporter"; -import {type GithubToken} from "../plugins/github/token"; -import {TimelineCred} from "../analysis/timeline/timelineCred"; -import {defaultParams, partialParams} from "../analysis/timeline/params"; -import {type TimelineCredParameters} from "../analysis/timeline/params"; - import {type Project} from "../core/project"; -import {setupProjectDirectory} from "../core/project_io"; -import { - type PluginDeclaration, - toJSON as pluginsToJSON, -} from "../analysis/pluginDeclaration"; -import * as Discourse from "../plugins/discourse/loadWeightedGraph"; -import * as Github from "../plugins/github/loadWeightedGraph"; -import * as WeightedGraph from "../core/weightedGraph"; import {type Weights as WeightsT} from "../core/weights"; -import {loadWeightedGraph} from "./loadWeightedGraph"; -import {DataDirectory} from "../backend/dataDirectory"; +import {type PluginDeclaration} from "../analysis/pluginDeclaration"; +import {type TimelineCredParameters} from "../analysis/timeline/params"; +import {type GithubToken} from "../plugins/github/token"; import {type CacheProvider} from "../backend/cache"; +import {DataDirectory} from "../backend/dataDirectory"; +import {TaskReporter} from "../util/taskReporter"; import {LoadContext} from "../backend/loadContext"; export type LoadOptions = {| @@ -34,7 +19,10 @@ export type LoadOptions = {| +githubToken: ?GithubToken, |}; -export async function loadContext( +/** + * Loads and computes cred for a Project, storing the result in a DataDirectory. + */ +export async function load( options: LoadOptions, reporter: TaskReporter ): Promise { @@ -46,104 +34,14 @@ export async function loadContext( weightsOverrides, } = options; const data = new DataDirectory(sourcecredDirectory); - const context = new LoadContext({cache: data, githubToken, reporter}); + const context = new LoadContext({ + cache: (data: CacheProvider), + githubToken, + reporter, + }); const result = await context.load(project, { params: params || {}, weightsOverrides, }); data.storeProject(project, result); } - -/** - * Loads and computes cred for a project. - * - * Loads the combined Graph for the specified project, saves it to disk, - * and computes cred for it using the provided TimelineCredParameters. - * - * A project directory will be created for the given project within the - * provided sourcecredDirectory, using the APIs in core/project_io. Within this - * project directory, there will be a `cred.json` file containing filtered - * timeline cred, and a `weightedGraph.json` file containing the combined graph. - * - * In the future, we should consider splitting this into cleaner, more atomic - * APIs (e.g. one for loading the graph; another for computing cred). - */ -export async function load( - options: LoadOptions, - taskReporter: TaskReporter -): Promise { - const { - project, - params, - plugins, - sourcecredDirectory, - githubToken, - weightsOverrides, - } = options; - const {identities, discourseServer} = project; - const fullParams = params == null ? defaultParams() : partialParams(params); - const loadTask = `load-${options.project.id}`; - taskReporter.start(loadTask); - const dataDirectory = new DataDirectory(sourcecredDirectory); - const cacheDirectory = path.join(sourcecredDirectory, "cache"); - await fs.mkdirp(cacheDirectory); - - let discourseOptions: ?Discourse.Options; - if (discourseServer != null) { - discourseOptions = { - discourseServer, - cacheDirectory, - }; - } - - let githubOptions: ?Github.Options; - if (project.repoIds.length) { - if (githubToken == null) { - throw new Error("Tried to load GitHub, but no GitHub token set."); - } - githubOptions = { - repoIds: project.repoIds, - token: githubToken, - cache: (dataDirectory: CacheProvider), - }; - } - - const identitySpec = { - identities, - discourseServerUrl: - discourseServer == null ? null : discourseServer.serverUrl, - }; - const weightedGraph = await loadWeightedGraph( - { - discourseOptions, - githubOptions, - identitySpec, - weightsOverrides, - }, - taskReporter - ); - - const projectDirectory = await setupProjectDirectory( - project, - sourcecredDirectory - ); - const graphFile = path.join(projectDirectory, "weightedGraph.json"); - const graphJSON = WeightedGraph.toJSON(weightedGraph); - await fs.writeFile(graphFile, stringify(graphJSON)); - - const pluginsFile = path.join(projectDirectory, "pluginDeclarations.json"); - const pluginsJSON = pluginsToJSON(plugins); - await fs.writeFile(pluginsFile, stringify(pluginsJSON)); - - taskReporter.start("compute-cred"); - const cred = await TimelineCred.compute({ - weightedGraph, - params: fullParams, - plugins, - }); - const credJSON = cred.toJSON(); - const credFile = path.join(projectDirectory, "cred.json"); - await fs.writeFile(credFile, stringify(credJSON)); - taskReporter.finish("compute-cred"); - taskReporter.finish(loadTask); -} diff --git a/src/api/load.test.js b/src/api/load.test.js deleted file mode 100644 index 472508b..0000000 --- a/src/api/load.test.js +++ /dev/null @@ -1,253 +0,0 @@ -// @flow - -import deepFreeze from "deep-freeze"; -import tmp from "tmp"; -import path from "path"; -import fs from "fs-extra"; - -import {validateToken} from "../plugins/github/token"; -import type {Options as LoadGraphOptions} from "../plugins/github/loadGraph"; -import type {Options as LoadDiscourseOptions} from "../plugins/discourse/loadDiscourse"; -import {contractIdentities} from "../plugins/identity/contractIdentities"; -import {type Project, createProject} from "../core/project"; -import { - directoryForProjectId, - getProjectIds, - loadProject, -} from "../core/project_io"; -import {makeRepoId} from "../plugins/github/repoId"; -import * as Weights from "../core/weights"; -import {Graph} from "../core/graph"; -import {node} from "../core/graphTestUtil"; -import {TestTaskReporter} from "../util/taskReporter"; -import {load, type LoadOptions} from "./load"; -import { - type TimelineCredParameters, - partialParams, -} from "../analysis/timeline/params"; -import * as WeightedGraph from "../core/weightedGraph"; -import {DataDirectory} from "../backend/dataDirectory"; -import {fromJSON as pluginsFromJSON} from "../analysis/pluginDeclaration"; - -type JestMockFn = $Call; -jest.mock("../plugins/github/loadWeightedGraph", () => ({ - loadWeightedGraph: jest.fn(), -})); -const githubWeightedGraph: JestMockFn = (require("../plugins/github/loadWeightedGraph") - .loadWeightedGraph: any); -jest.mock("../plugins/discourse/loadWeightedGraph", () => ({ - loadWeightedGraph: jest.fn(), -})); -const discourseWeightedGraph: JestMockFn = (require("../plugins/discourse/loadWeightedGraph") - .loadWeightedGraph: any); - -jest.mock("../analysis/timeline/timelineCred", () => ({ - TimelineCred: {compute: jest.fn()}, -})); -const timelineCredCompute: JestMockFn = (require("../analysis/timeline/timelineCred") - .TimelineCred.compute: any); - -describe("api/load", () => { - const exampleGithubToken = validateToken("0".repeat(40)); - const fakeTimelineCred = deepFreeze({ - toJSON: () => ({is: "fake-timeline-cred"}), - }); - const githubSentinel = node("github-sentinel"); - const githubGraph = () => { - const graph = new Graph().addNode(githubSentinel); - return {graph, weights: Weights.empty()}; - }; - const discourseSentinel = node("discourse-sentinel"); - const discourseGraph = () => { - const graph = new Graph().addNode(discourseSentinel); - return {graph, weights: Weights.empty()}; - }; - const combinedGraph = () => - WeightedGraph.merge([githubGraph(), discourseGraph()]); - beforeEach(() => { - jest.clearAllMocks(); - githubWeightedGraph.mockResolvedValue(githubGraph()); - discourseWeightedGraph.mockResolvedValue(discourseGraph()); - timelineCredCompute.mockResolvedValue(fakeTimelineCred); - }); - const discourseServerUrl = "https://example.com"; - const project: Project = createProject({ - id: "foo", - repoIds: [makeRepoId("foo", "bar")], - discourseServer: {serverUrl: discourseServerUrl}, - }); - deepFreeze(project); - const weightsOverrides = Weights.empty(); - const params: $Shape = {}; - const plugins = deepFreeze([]); - const example = () => { - const sourcecredDirectory = tmp.dirSync().name; - const taskReporter = new TestTaskReporter(); - const options: LoadOptions = { - sourcecredDirectory, - githubToken: exampleGithubToken, - params, - plugins, - project, - weightsOverrides, - }; - return {options, taskReporter, sourcecredDirectory}; - }; - - it("sets up a project directory for the project", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - await load(options, taskReporter); - expect(await getProjectIds(sourcecredDirectory)).toEqual([project.id]); - expect(await loadProject(project.id, sourcecredDirectory)).toEqual(project); - }); - - it("calls github githubWeightedGraph with the right options", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - await load(options, taskReporter); - const cache = new DataDirectory(sourcecredDirectory); - const expectedLoadGraphOptions: LoadGraphOptions = { - repoIds: project.repoIds, - token: exampleGithubToken, - cache, - }; - expect(githubWeightedGraph).toHaveBeenCalledWith( - expectedLoadGraphOptions, - taskReporter - ); - }); - - it("calls discourseWeightedGraph with the right options", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - await load(options, taskReporter); - const cacheDirectory = path.join(sourcecredDirectory, "cache"); - const expectedOptions: LoadDiscourseOptions = { - discourseServer: {serverUrl: discourseServerUrl}, - cacheDirectory, - }; - expect(discourseWeightedGraph).toHaveBeenCalledWith( - expectedOptions, - taskReporter - ); - }); - - it("saves a merged graph to disk", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - await load(options, taskReporter); - const projectDirectory = directoryForProjectId( - project.id, - sourcecredDirectory - ); - const graphFile = path.join(projectDirectory, "weightedGraph.json"); - const graphJSON = JSON.parse(await fs.readFile(graphFile)); - const expectedJSON = WeightedGraph.toJSON(combinedGraph()); - expect(graphJSON).toEqual(expectedJSON); - }); - - it("calls TimelineCred.compute with the right graph and options", async () => { - const {options, taskReporter} = example(); - await load(options, taskReporter); - const args = timelineCredCompute.mock.calls[0][0]; - expect(args.weightedGraph.graph.equals(combinedGraph().graph)).toBe(true); - expect(args.weightedGraph.weights).toEqual(combinedGraph().weights); - expect(args.params).toEqual(partialParams(params)); - expect(args.plugins).toEqual(plugins); - }); - - it("saves the resultant cred.json to disk", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - await load(options, taskReporter); - const projectDirectory = directoryForProjectId( - project.id, - sourcecredDirectory - ); - const credFile = path.join(projectDirectory, "cred.json"); - const credJSON = JSON.parse(await fs.readFile(credFile)); - expect(credJSON).toEqual(fakeTimelineCred.toJSON()); - }); - - it("gives the right tasks to the TaskReporter", async () => { - const {options, taskReporter} = example(); - await load(options, taskReporter); - expect(taskReporter.activeTasks()).toEqual([]); - expect(taskReporter.entries()).toEqual([ - {type: "START", taskId: "load-foo"}, - {type: "START", taskId: "load-weighted-graph"}, - {type: "FINISH", taskId: "load-weighted-graph"}, - {type: "START", taskId: "compute-cred"}, - {type: "FINISH", taskId: "compute-cred"}, - {type: "FINISH", taskId: "load-foo"}, - ]); - }); - - it("errors if GitHub repoIds are provided without a GitHub token", () => { - const {options, taskReporter} = example(); - const optionsWithoutToken = {...options, githubToken: null}; - expect.assertions(1); - return load(optionsWithoutToken, taskReporter).catch((e) => - expect(e.message).toMatch("no GitHub token") - ); - }); - - it("only loads GitHub if no Discourse server set", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - const newProject = {...options.project, discourseServer: null}; - const newOptions = {...options, project: newProject}; - await load(newOptions, taskReporter); - expect(discourseWeightedGraph).not.toHaveBeenCalled(); - const projectDirectory = directoryForProjectId( - project.id, - sourcecredDirectory - ); - const graphFile = path.join(projectDirectory, "weightedGraph.json"); - const graphJSON = JSON.parse(await fs.readFile(graphFile)); - const expectedJSON = WeightedGraph.toJSON(githubGraph()); - expect(graphJSON).toEqual(expectedJSON); - }); - - it("only loads Discourse if no GitHub repoIds set ", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - const newProject = {...options.project, repoIds: []}; - const newOptions = {...options, project: newProject, githubToken: null}; - await load(newOptions, taskReporter); - expect(githubWeightedGraph).not.toHaveBeenCalled(); - const projectDirectory = directoryForProjectId( - project.id, - sourcecredDirectory - ); - const graphFile = path.join(projectDirectory, "weightedGraph.json"); - const graphJSON = JSON.parse(await fs.readFile(graphFile)); - const expectedJSON = WeightedGraph.toJSON(discourseGraph()); - expect(graphJSON).toEqual(expectedJSON); - }); - - it("applies identity transformations, if present in the project", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - const identity = {username: "identity", aliases: []}; - const newProject = {...options.project, identities: [identity]}; - const newOptions = {...options, project: newProject}; - await load(newOptions, taskReporter); - const projectDirectory = directoryForProjectId( - project.id, - sourcecredDirectory - ); - const graphFile = path.join(projectDirectory, "weightedGraph.json"); - const graphJSON = JSON.parse(await fs.readFile(graphFile)); - const identitySpec = {identities: [identity], discourseServerUrl}; - const identityGraph = contractIdentities(combinedGraph(), identitySpec); - const expectedJSON = WeightedGraph.toJSON(identityGraph); - expect(graphJSON).toEqual(expectedJSON); - }); - - it("saves plugin declarations to disk", async () => { - const {options, taskReporter, sourcecredDirectory} = example(); - await load(options, taskReporter); - const projectDirectory = directoryForProjectId( - project.id, - sourcecredDirectory - ); - const pluginsFile = path.join(projectDirectory, "pluginDeclarations.json"); - const pluginsJSON = JSON.parse(await fs.readFile(pluginsFile)); - const actualPlugins = pluginsFromJSON(pluginsJSON); - expect(actualPlugins).toEqual(plugins); - }); -}); diff --git a/src/api/loadWeightedGraph.js b/src/api/loadWeightedGraph.js deleted file mode 100644 index 2f5f6ea..0000000 --- a/src/api/loadWeightedGraph.js +++ /dev/null @@ -1,71 +0,0 @@ -// @flow - -import * as WeightedGraph from "../core/weightedGraph"; -import {type WeightedGraph as WeightedGraphT} from "../core/weightedGraph"; -import {type Weights as WeightsT} from "../core/weights"; -import {type NodeContraction} from "../core/graph"; -import {TaskReporter} from "../util/taskReporter"; -import {type IdentitySpec} from "../plugins/identity/identity"; -import {contractWeightedGraph} from "../plugins/identity/contractIdentities"; -import {nodeContractions} from "../plugins/identity/nodeContractions"; -import * as Discourse from "../plugins/discourse/loadWeightedGraph"; -import * as Github from "../plugins/github/loadWeightedGraph"; - -export type LoadWeightedGraphOptions = {| - +discourseOptions: ?Discourse.Options, - +githubOptions: ?Github.Options, - +identitySpec: IdentitySpec, - +weightsOverrides: WeightsT, -|}; - -export async function loadWeightedGraph( - options: LoadWeightedGraphOptions, - taskReporter: TaskReporter -): Promise { - taskReporter.start("load-weighted-graph"); - const { - discourseOptions, - githubOptions, - identitySpec, - weightsOverrides, - } = options; - const pluginGraphs = await _loadPluginGraphs( - discourseOptions, - githubOptions, - taskReporter - ); - const contractions = nodeContractions(identitySpec); - const result = _combineGraphs(pluginGraphs, contractions, weightsOverrides); - taskReporter.finish("load-weighted-graph"); - return result; -} - -export function _loadPluginGraphs( - discourseOptions: ?Discourse.Options, - githubOptions: ?Github.Options, - taskReporter: TaskReporter -): Promise<$ReadOnlyArray> { - const promises: Promise[] = []; - if (discourseOptions) { - const promise = Discourse.loadWeightedGraph(discourseOptions, taskReporter); - promises.push(promise); - } - if (githubOptions) { - const promise = Github.loadWeightedGraph(githubOptions, taskReporter); - promises.push(promise); - } - // It's important to use Promise.all so that we can load the plugins in - // parallel -- since loading is often IO-bound, this can be a big performance - // improvement. - return Promise.all(promises); -} - -export function _combineGraphs( - graphs: $ReadOnlyArray, - contractions: $ReadOnlyArray, - weightsOverrides: WeightsT -): WeightedGraphT { - const merged = WeightedGraph.merge(graphs); - const contracted = contractWeightedGraph(merged, contractions); - return WeightedGraph.overrideWeights(contracted, weightsOverrides); -} diff --git a/src/api/loadWeightedGraph.test.js b/src/api/loadWeightedGraph.test.js deleted file mode 100644 index 3e43af1..0000000 --- a/src/api/loadWeightedGraph.test.js +++ /dev/null @@ -1,51 +0,0 @@ -// @flow - -import {node} from "../core/graphTestUtil"; -import * as WeightedGraph from "../core/weightedGraph"; -import * as Weights from "../core/weights"; -import {_combineGraphs} from "./loadWeightedGraph"; - -describe("api/loadWeightedGraph", () => { - // The _combineGraphs subfunction does the "interesting" work here; the - // rest is just composing IO heavy stuff (e.g. actually generating the GitHub/ - // Discourse graphs). - describe("_combineGraphs", () => { - const foo = node("foo"); - const bar = node("bar"); - const zod = node("zod"); - it("merges the input graphs", () => { - const wg1 = WeightedGraph.empty(); - const wg2 = WeightedGraph.empty(); - wg1.graph.addNode(foo); - wg1.weights.nodeWeights.set(foo.address, 3); - wg2.graph.addNode(bar); - wg2.weights.nodeWeights.set(bar.address, 3); - const expected = WeightedGraph.merge([wg1, wg2]); - expect(_combineGraphs([wg1, wg2], [], Weights.empty())).toEqual(expected); - }); - it("uses the provided contractions", () => { - const wg = WeightedGraph.empty(); - wg.graph.addNode(foo); - wg.graph.addNode(bar); - const contraction = {old: [foo.address, bar.address], replacement: zod}; - const expected = WeightedGraph.empty(); - expected.graph.addNode(zod); - const combined = _combineGraphs([wg], [contraction], Weights.empty()); - expect(combined).toEqual(expected); - }); - it("uses the weights as overrides", () => { - const wg = WeightedGraph.empty(); - wg.weights.nodeWeights.set(foo.address, 3); - wg.weights.nodeWeights.set(bar.address, 3); - const weights = Weights.empty(); - weights.nodeWeights.set(foo.address, 5); - weights.nodeWeights.set(zod.address, 5); - const combined = _combineGraphs([wg], [], weights); - const expected = WeightedGraph.empty(); - expected.weights.nodeWeights.set(bar.address, 3); - expected.weights.nodeWeights.set(foo.address, 5); - expected.weights.nodeWeights.set(zod.address, 5); - expect(expected).toEqual(combined); - }); - }); -}); diff --git a/src/cli/discourse.js b/src/cli/discourse.js index c082385..11be5a1 100644 --- a/src/cli/discourse.js +++ b/src/cli/discourse.js @@ -8,7 +8,7 @@ import {LoggingTaskReporter} from "../util/taskReporter"; import type {Command} from "./command"; import * as Common from "./common"; import * as Weights from "../core/weights"; -import {loadContext as load} from "../api/load"; +import {load} from "../api/load"; import {declaration as discourseDeclaration} from "../plugins/discourse/declaration"; import {type Project, createProject} from "../core/project"; diff --git a/src/cli/load.js b/src/cli/load.js index 532704d..bf920d2 100644 --- a/src/cli/load.js +++ b/src/cli/load.js @@ -7,7 +7,7 @@ import type {Command} from "./command"; import * as Common from "./common"; import * as Weights from "../core/weights"; import {projectFromJSON} from "../core/project"; -import {loadContext as load} from "../api/load"; +import {load} from "../api/load"; import {specToProject} from "../plugins/github/specToProject"; import fs from "fs-extra"; import {type PluginDeclaration} from "../analysis/pluginDeclaration"; diff --git a/src/cli/load.test.js b/src/cli/load.test.js index 4916aee..9dd8c76 100644 --- a/src/cli/load.test.js +++ b/src/cli/load.test.js @@ -16,9 +16,9 @@ import {makeRepoId, stringToRepoId} from "../plugins/github/repoId"; import {validateToken} from "../plugins/github/token"; import {defaultParams} from "../analysis/timeline/params"; -jest.mock("../api/load", () => ({loadContext: jest.fn()})); +jest.mock("../api/load", () => ({load: jest.fn()})); type JestMockFn = $Call; -const load: JestMockFn = (require("../api/load").loadContext: any); +const load: JestMockFn = (require("../api/load").load: any); describe("cli/load", () => { const exampleGithubToken = validateToken("0".repeat(40));