status-im
/
sourcecred
mirror of https://github.com/status-im/sourcecred.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Remove deprecated load function (#1630) 

One of several cleanup commits. See #1629.

The previously created loadContext (see #1622) function will be used
instead and renamed to be load.

We're removing significant amounts of test code as well. The individual
components that make up the replacement function already provide the
needed coverage, like `dataDirectory.test.js` and `loadContext.test.js`.
The integration provided by this function is also covered through the
sharness `test_load_example_github.t`.
This commit is contained in:
Robin van Boven 2020-02-05 11:27:43 +01:00 committed by GitHub
parent 361961c59f
commit e48902cfc9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 18 additions and 495 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
src/api/load.js
View File

@ -1,28 +1,13 @@
// @flow
import fs from "fs-extra";
import path from "path";
import stringify from "json-stable-stringify";
import {TaskReporter} from "../util/taskReporter";
import {type GithubToken} from "../plugins/github/token";
import {TimelineCred} from "../analysis/timeline/timelineCred";
import {defaultParams, partialParams} from "../analysis/timeline/params";
import {type TimelineCredParameters} from "../analysis/timeline/params";
import {type Project} from "../core/project";
import {setupProjectDirectory} from "../core/project_io";
import {
type PluginDeclaration,
toJSON as pluginsToJSON,
} from "../analysis/pluginDeclaration";
import * as Discourse from "../plugins/discourse/loadWeightedGraph";
import * as Github from "../plugins/github/loadWeightedGraph";
import * as WeightedGraph from "../core/weightedGraph";
import {type Weights as WeightsT} from "../core/weights";
import {loadWeightedGraph} from "./loadWeightedGraph";
import {DataDirectory} from "../backend/dataDirectory";
import {type PluginDeclaration} from "../analysis/pluginDeclaration";
import {type TimelineCredParameters} from "../analysis/timeline/params";
import {type GithubToken} from "../plugins/github/token";
import {type CacheProvider} from "../backend/cache";
import {DataDirectory} from "../backend/dataDirectory";
import {TaskReporter} from "../util/taskReporter";
import {LoadContext} from "../backend/loadContext";
export type LoadOptions = {|
@ -34,7 +19,10 @@ export type LoadOptions = {|
+githubToken: ?GithubToken,
|};
export async function loadContext(
/**
* Loads and computes cred for a Project, storing the result in a DataDirectory.
*/
export async function load(
options: LoadOptions,
reporter: TaskReporter
): Promise<void> {
@ -46,104 +34,14 @@ export async function loadContext(
weightsOverrides,
} = options;
const data = new DataDirectory(sourcecredDirectory);
const context = new LoadContext({cache: data, githubToken, reporter});
const context = new LoadContext({
cache: (data: CacheProvider),
githubToken,
reporter,
});
const result = await context.load(project, {
params: params || {},
weightsOverrides,
});
data.storeProject(project, result);
}
/**
* Loads and computes cred for a project.
*
* Loads the combined Graph for the specified project, saves it to disk,
* and computes cred for it using the provided TimelineCredParameters.
*
* A project directory will be created for the given project within the
* provided sourcecredDirectory, using the APIs in core/project_io. Within this
* project directory, there will be a `cred.json` file containing filtered
* timeline cred, and a `weightedGraph.json` file containing the combined graph.
*
* In the future, we should consider splitting this into cleaner, more atomic
* APIs (e.g. one for loading the graph; another for computing cred).
*/
export async function load(
options: LoadOptions,
taskReporter: TaskReporter
): Promise<void> {
const {
project,
params,
plugins,
sourcecredDirectory,
githubToken,
weightsOverrides,
} = options;
const {identities, discourseServer} = project;
const fullParams = params == null ? defaultParams() : partialParams(params);
const loadTask = `load-${options.project.id}`;
taskReporter.start(loadTask);
const dataDirectory = new DataDirectory(sourcecredDirectory);
const cacheDirectory = path.join(sourcecredDirectory, "cache");
await fs.mkdirp(cacheDirectory);
let discourseOptions: ?Discourse.Options;
if (discourseServer != null) {
discourseOptions = {
discourseServer,
cacheDirectory,
};
}
let githubOptions: ?Github.Options;
if (project.repoIds.length) {
if (githubToken == null) {
throw new Error("Tried to load GitHub, but no GitHub token set.");
}
githubOptions = {
repoIds: project.repoIds,
token: githubToken,
cache: (dataDirectory: CacheProvider),
};
}
const identitySpec = {
identities,
discourseServerUrl:
discourseServer == null ? null : discourseServer.serverUrl,
};
const weightedGraph = await loadWeightedGraph(
{
discourseOptions,
githubOptions,
identitySpec,
weightsOverrides,
},
taskReporter
);
const projectDirectory = await setupProjectDirectory(
project,
sourcecredDirectory
);
const graphFile = path.join(projectDirectory, "weightedGraph.json");
const graphJSON = WeightedGraph.toJSON(weightedGraph);
await fs.writeFile(graphFile, stringify(graphJSON));
const pluginsFile = path.join(projectDirectory, "pluginDeclarations.json");
const pluginsJSON = pluginsToJSON(plugins);
await fs.writeFile(pluginsFile, stringify(pluginsJSON));
taskReporter.start("compute-cred");
const cred = await TimelineCred.compute({
weightedGraph,
params: fullParams,
plugins,
});
const credJSON = cred.toJSON();
const credFile = path.join(projectDirectory, "cred.json");
await fs.writeFile(credFile, stringify(credJSON));
taskReporter.finish("compute-cred");
taskReporter.finish(loadTask);
}

253
src/api/load.test.js
View File

@ -1,253 +0,0 @@
// @flow
import deepFreeze from "deep-freeze";
import tmp from "tmp";
import path from "path";
import fs from "fs-extra";
import {validateToken} from "../plugins/github/token";
import type {Options as LoadGraphOptions} from "../plugins/github/loadGraph";
import type {Options as LoadDiscourseOptions} from "../plugins/discourse/loadDiscourse";
import {contractIdentities} from "../plugins/identity/contractIdentities";
import {type Project, createProject} from "../core/project";
import {
directoryForProjectId,
getProjectIds,
loadProject,
} from "../core/project_io";
import {makeRepoId} from "../plugins/github/repoId";
import * as Weights from "../core/weights";
import {Graph} from "../core/graph";
import {node} from "../core/graphTestUtil";
import {TestTaskReporter} from "../util/taskReporter";
import {load, type LoadOptions} from "./load";
import {
type TimelineCredParameters,
partialParams,
} from "../analysis/timeline/params";
import * as WeightedGraph from "../core/weightedGraph";
import {DataDirectory} from "../backend/dataDirectory";
import {fromJSON as pluginsFromJSON} from "../analysis/pluginDeclaration";
type JestMockFn = $Call<typeof jest.fn>;
jest.mock("../plugins/github/loadWeightedGraph", () => ({
loadWeightedGraph: jest.fn(),
}));
const githubWeightedGraph: JestMockFn = (require("../plugins/github/loadWeightedGraph")
.loadWeightedGraph: any);
jest.mock("../plugins/discourse/loadWeightedGraph", () => ({
loadWeightedGraph: jest.fn(),
}));
const discourseWeightedGraph: JestMockFn = (require("../plugins/discourse/loadWeightedGraph")
.loadWeightedGraph: any);
jest.mock("../analysis/timeline/timelineCred", () => ({
TimelineCred: {compute: jest.fn()},
}));
const timelineCredCompute: JestMockFn = (require("../analysis/timeline/timelineCred")
.TimelineCred.compute: any);
describe("api/load", () => {
const exampleGithubToken = validateToken("0".repeat(40));
const fakeTimelineCred = deepFreeze({
toJSON: () => ({is: "fake-timeline-cred"}),
});
const githubSentinel = node("github-sentinel");
const githubGraph = () => {
const graph = new Graph().addNode(githubSentinel);
return {graph, weights: Weights.empty()};
};
const discourseSentinel = node("discourse-sentinel");
const discourseGraph = () => {
const graph = new Graph().addNode(discourseSentinel);
return {graph, weights: Weights.empty()};
};
const combinedGraph = () =>
WeightedGraph.merge([githubGraph(), discourseGraph()]);
beforeEach(() => {
jest.clearAllMocks();
githubWeightedGraph.mockResolvedValue(githubGraph());
discourseWeightedGraph.mockResolvedValue(discourseGraph());
timelineCredCompute.mockResolvedValue(fakeTimelineCred);
});
const discourseServerUrl = "https://example.com";
const project: Project = createProject({
id: "foo",
repoIds: [makeRepoId("foo", "bar")],
discourseServer: {serverUrl: discourseServerUrl},
});
deepFreeze(project);
const weightsOverrides = Weights.empty();
const params: $Shape<TimelineCredParameters> = {};
const plugins = deepFreeze([]);
const example = () => {
const sourcecredDirectory = tmp.dirSync().name;
const taskReporter = new TestTaskReporter();
const options: LoadOptions = {
sourcecredDirectory,
githubToken: exampleGithubToken,
params,
plugins,
project,
weightsOverrides,
};
return {options, taskReporter, sourcecredDirectory};
};
it("sets up a project directory for the project", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
await load(options, taskReporter);
expect(await getProjectIds(sourcecredDirectory)).toEqual([project.id]);
expect(await loadProject(project.id, sourcecredDirectory)).toEqual(project);
});
it("calls github githubWeightedGraph with the right options", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
await load(options, taskReporter);
const cache = new DataDirectory(sourcecredDirectory);
const expectedLoadGraphOptions: LoadGraphOptions = {
repoIds: project.repoIds,
token: exampleGithubToken,
cache,
};
expect(githubWeightedGraph).toHaveBeenCalledWith(
expectedLoadGraphOptions,
taskReporter
);
});
it("calls discourseWeightedGraph with the right options", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
await load(options, taskReporter);
const cacheDirectory = path.join(sourcecredDirectory, "cache");
const expectedOptions: LoadDiscourseOptions = {
discourseServer: {serverUrl: discourseServerUrl},
cacheDirectory,
};
expect(discourseWeightedGraph).toHaveBeenCalledWith(
expectedOptions,
taskReporter
);
});
it("saves a merged graph to disk", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
await load(options, taskReporter);
const projectDirectory = directoryForProjectId(
project.id,
sourcecredDirectory
);
const graphFile = path.join(projectDirectory, "weightedGraph.json");
const graphJSON = JSON.parse(await fs.readFile(graphFile));
const expectedJSON = WeightedGraph.toJSON(combinedGraph());
expect(graphJSON).toEqual(expectedJSON);
});
it("calls TimelineCred.compute with the right graph and options", async () => {
const {options, taskReporter} = example();
await load(options, taskReporter);
const args = timelineCredCompute.mock.calls[0][0];
expect(args.weightedGraph.graph.equals(combinedGraph().graph)).toBe(true);
expect(args.weightedGraph.weights).toEqual(combinedGraph().weights);
expect(args.params).toEqual(partialParams(params));
expect(args.plugins).toEqual(plugins);
});
it("saves the resultant cred.json to disk", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
await load(options, taskReporter);
const projectDirectory = directoryForProjectId(
project.id,
sourcecredDirectory
);
const credFile = path.join(projectDirectory, "cred.json");
const credJSON = JSON.parse(await fs.readFile(credFile));
expect(credJSON).toEqual(fakeTimelineCred.toJSON());
});
it("gives the right tasks to the TaskReporter", async () => {
const {options, taskReporter} = example();
await load(options, taskReporter);
expect(taskReporter.activeTasks()).toEqual([]);
expect(taskReporter.entries()).toEqual([
{type: "START", taskId: "load-foo"},
{type: "START", taskId: "load-weighted-graph"},
{type: "FINISH", taskId: "load-weighted-graph"},
{type: "START", taskId: "compute-cred"},
{type: "FINISH", taskId: "compute-cred"},
{type: "FINISH", taskId: "load-foo"},
]);
});
it("errors if GitHub repoIds are provided without a GitHub token", () => {
const {options, taskReporter} = example();
const optionsWithoutToken = {...options, githubToken: null};
expect.assertions(1);
return load(optionsWithoutToken, taskReporter).catch((e) =>
expect(e.message).toMatch("no GitHub token")
);
});
it("only loads GitHub if no Discourse server set", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
const newProject = {...options.project, discourseServer: null};
const newOptions = {...options, project: newProject};
await load(newOptions, taskReporter);
expect(discourseWeightedGraph).not.toHaveBeenCalled();
const projectDirectory = directoryForProjectId(
project.id,
sourcecredDirectory
);
const graphFile = path.join(projectDirectory, "weightedGraph.json");
const graphJSON = JSON.parse(await fs.readFile(graphFile));
const expectedJSON = WeightedGraph.toJSON(githubGraph());
expect(graphJSON).toEqual(expectedJSON);
});
it("only loads Discourse if no GitHub repoIds set ", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
const newProject = {...options.project, repoIds: []};
const newOptions = {...options, project: newProject, githubToken: null};
await load(newOptions, taskReporter);
expect(githubWeightedGraph).not.toHaveBeenCalled();
const projectDirectory = directoryForProjectId(
project.id,
sourcecredDirectory
);
const graphFile = path.join(projectDirectory, "weightedGraph.json");
const graphJSON = JSON.parse(await fs.readFile(graphFile));
const expectedJSON = WeightedGraph.toJSON(discourseGraph());
expect(graphJSON).toEqual(expectedJSON);
});
it("applies identity transformations, if present in the project", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
const identity = {username: "identity", aliases: []};
const newProject = {...options.project, identities: [identity]};
const newOptions = {...options, project: newProject};
await load(newOptions, taskReporter);
const projectDirectory = directoryForProjectId(
project.id,
sourcecredDirectory
);
const graphFile = path.join(projectDirectory, "weightedGraph.json");
const graphJSON = JSON.parse(await fs.readFile(graphFile));
const identitySpec = {identities: [identity], discourseServerUrl};
const identityGraph = contractIdentities(combinedGraph(), identitySpec);
const expectedJSON = WeightedGraph.toJSON(identityGraph);
expect(graphJSON).toEqual(expectedJSON);
});
it("saves plugin declarations to disk", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
await load(options, taskReporter);
const projectDirectory = directoryForProjectId(
project.id,
sourcecredDirectory
);
const pluginsFile = path.join(projectDirectory, "pluginDeclarations.json");
const pluginsJSON = JSON.parse(await fs.readFile(pluginsFile));
const actualPlugins = pluginsFromJSON(pluginsJSON);
expect(actualPlugins).toEqual(plugins);
});
});

71
src/api/loadWeightedGraph.js
View File

@ -1,71 +0,0 @@
// @flow
import * as WeightedGraph from "../core/weightedGraph";
import {type WeightedGraph as WeightedGraphT} from "../core/weightedGraph";
import {type Weights as WeightsT} from "../core/weights";
import {type NodeContraction} from "../core/graph";
import {TaskReporter} from "../util/taskReporter";
import {type IdentitySpec} from "../plugins/identity/identity";
import {contractWeightedGraph} from "../plugins/identity/contractIdentities";
import {nodeContractions} from "../plugins/identity/nodeContractions";
import * as Discourse from "../plugins/discourse/loadWeightedGraph";
import * as Github from "../plugins/github/loadWeightedGraph";
export type LoadWeightedGraphOptions = {|
+discourseOptions: ?Discourse.Options,
+githubOptions: ?Github.Options,
+identitySpec: IdentitySpec,
+weightsOverrides: WeightsT,
|};
export async function loadWeightedGraph(
options: LoadWeightedGraphOptions,
taskReporter: TaskReporter
): Promise<WeightedGraphT> {
taskReporter.start("load-weighted-graph");
const {
discourseOptions,
githubOptions,
identitySpec,
weightsOverrides,
} = options;
const pluginGraphs = await _loadPluginGraphs(
discourseOptions,
githubOptions,
taskReporter
);
const contractions = nodeContractions(identitySpec);
const result = _combineGraphs(pluginGraphs, contractions, weightsOverrides);
taskReporter.finish("load-weighted-graph");
return result;
}
export function _loadPluginGraphs(
discourseOptions: ?Discourse.Options,
githubOptions: ?Github.Options,
taskReporter: TaskReporter
): Promise<$ReadOnlyArray<WeightedGraphT>> {
const promises: Promise<WeightedGraphT>[] = [];
if (discourseOptions) {
const promise = Discourse.loadWeightedGraph(discourseOptions, taskReporter);
promises.push(promise);
}
if (githubOptions) {
const promise = Github.loadWeightedGraph(githubOptions, taskReporter);
promises.push(promise);
}
// It's important to use Promise.all so that we can load the plugins in
// parallel -- since loading is often IO-bound, this can be a big performance
// improvement.
return Promise.all(promises);
}
export function _combineGraphs(
graphs: $ReadOnlyArray<WeightedGraphT>,
contractions: $ReadOnlyArray<NodeContraction>,
weightsOverrides: WeightsT
): WeightedGraphT {
const merged = WeightedGraph.merge(graphs);
const contracted = contractWeightedGraph(merged, contractions);
return WeightedGraph.overrideWeights(contracted, weightsOverrides);
}

51
src/api/loadWeightedGraph.test.js
View File

@ -1,51 +0,0 @@
// @flow
import {node} from "../core/graphTestUtil";
import * as WeightedGraph from "../core/weightedGraph";
import * as Weights from "../core/weights";
import {_combineGraphs} from "./loadWeightedGraph";
describe("api/loadWeightedGraph", () => {
// The _combineGraphs subfunction does the "interesting" work here; the
// rest is just composing IO heavy stuff (e.g. actually generating the GitHub/
// Discourse graphs).
describe("_combineGraphs", () => {
const foo = node("foo");
const bar = node("bar");
const zod = node("zod");
it("merges the input graphs", () => {
const wg1 = WeightedGraph.empty();
const wg2 = WeightedGraph.empty();
wg1.graph.addNode(foo);
wg1.weights.nodeWeights.set(foo.address, 3);
wg2.graph.addNode(bar);
wg2.weights.nodeWeights.set(bar.address, 3);
const expected = WeightedGraph.merge([wg1, wg2]);
expect(_combineGraphs([wg1, wg2], [], Weights.empty())).toEqual(expected);
});
it("uses the provided contractions", () => {
const wg = WeightedGraph.empty();
wg.graph.addNode(foo);
wg.graph.addNode(bar);
const contraction = {old: [foo.address, bar.address], replacement: zod};
const expected = WeightedGraph.empty();
expected.graph.addNode(zod);
const combined = _combineGraphs([wg], [contraction], Weights.empty());
expect(combined).toEqual(expected);
});
it("uses the weights as overrides", () => {
const wg = WeightedGraph.empty();
wg.weights.nodeWeights.set(foo.address, 3);
wg.weights.nodeWeights.set(bar.address, 3);
const weights = Weights.empty();
weights.nodeWeights.set(foo.address, 5);
weights.nodeWeights.set(zod.address, 5);
const combined = _combineGraphs([wg], [], weights);
const expected = WeightedGraph.empty();
expected.weights.nodeWeights.set(bar.address, 3);
expected.weights.nodeWeights.set(foo.address, 5);
expected.weights.nodeWeights.set(zod.address, 5);
expect(expected).toEqual(combined);
});
});
});

2
src/cli/discourse.js
View File

@ -8,7 +8,7 @@ import {LoggingTaskReporter} from "../util/taskReporter";
import type {Command} from "./command";
import * as Common from "./common";
import * as Weights from "../core/weights";
import {loadContext as load} from "../api/load";
import {load} from "../api/load";
import {declaration as discourseDeclaration} from "../plugins/discourse/declaration";
import {type Project, createProject} from "../core/project";

2
src/cli/load.js
View File

@ -7,7 +7,7 @@ import type {Command} from "./command";
import * as Common from "./common";
import * as Weights from "../core/weights";
import {projectFromJSON} from "../core/project";
import {loadContext as load} from "../api/load";
import {load} from "../api/load";
import {specToProject} from "../plugins/github/specToProject";
import fs from "fs-extra";
import {type PluginDeclaration} from "../analysis/pluginDeclaration";

4
src/cli/load.test.js
View File

@ -16,9 +16,9 @@ import {makeRepoId, stringToRepoId} from "../plugins/github/repoId";
import {validateToken} from "../plugins/github/token";
import {defaultParams} from "../analysis/timeline/params";
jest.mock("../api/load", () => ({loadContext: jest.fn()}));
jest.mock("../api/load", () => ({load: jest.fn()}));
type JestMockFn = $Call<typeof jest.fn>;
const load: JestMockFn = (require("../api/load").loadContext: any);
const load: JestMockFn = (require("../api/load").load: any);
describe("cli/load", () => {
const exampleGithubToken = validateToken("0".repeat(40));