From aeaa945a27186e4ce85ed4af75749e74bb2c0149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Tue, 4 Feb 2020 10:30:14 -0800 Subject: [PATCH] api/load: save plugin declarations to disk (#1624) This builds on #1623 and is another step towards separating cred computation from plugin declarations, as described in #1557. Basically, this will allow the frontend to get plugin declarations even if the TimelineCred computation never saw them. This commit modifies `api/load`, and adds a new facility to `DataDirectory` for saving the PluginDeclarations (which will be used by @Beanow's in-flight refactor of `api/load`). Test plan: See included unit tests, also try loading a project and inspect the newlys saved file. --- .../pluginDeclarations.json | 1 + src/api/load.js | 9 ++++++++- src/api/load.test.js | 14 ++++++++++++++ src/backend/dataDirectory.js | 16 +++++++++++++--- src/backend/dataDirectory.test.js | 11 +++++++++++ src/backend/projectStorage.js | 2 ++ 6 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 sharness/__snapshots__/example-github-load/projects/c291cmNlY3JlZC10ZXN0L2V4YW1wbGUtZ2l0aHVi/pluginDeclarations.json diff --git a/sharness/__snapshots__/example-github-load/projects/c291cmNlY3JlZC10ZXN0L2V4YW1wbGUtZ2l0aHVi/pluginDeclarations.json b/sharness/__snapshots__/example-github-load/projects/c291cmNlY3JlZC10ZXN0L2V4YW1wbGUtZ2l0aHVi/pluginDeclarations.json new file mode 100644 index 0000000..456fd91 --- /dev/null +++ b/sharness/__snapshots__/example-github-load/projects/c291cmNlY3JlZC10ZXN0L2V4YW1wbGUtZ2l0aHVi/pluginDeclarations.json @@ -0,0 +1 @@ +[{"type":"sourcecred/pluginDeclarations","version":"0.1.0"},[{"edgePrefix":"E\u0000sourcecred\u0000github\u0000","edgeTypes":[{"backwardName":"is authored by","defaultWeight":{"backwards":1,"forwards":0.5},"description":"Connects a GitHub account to a post that they authored.\n\nExamples of posts include issues, pull requests, and comments.\n","forwardName":"authors","prefix":"E\u0000sourcecred\u0000github\u0000AUTHORS\u0000"},{"backwardName":"has child","defaultWeight":{"backwards":0.25,"forwards":1},"description":"Connects a GitHub entity to its child entities.\n\nFor example, a Repository has Issues and Pull Requests as children, and a\nPull Request has comments and reviews as children.\n","forwardName":"has parent","prefix":"E\u0000sourcecred\u0000github\u0000HAS_PARENT\u0000"},{"backwardName":"is merged by","defaultWeight":{"backwards":1,"forwards":0.5},"description":"Connects a GitHub pull request to the Git commit that it merges.\n","forwardName":"merges","prefix":"E\u0000sourcecred\u0000github\u0000MERGED_AS\u0000"},{"backwardName":"is referenced by","defaultWeight":{"backwards":0,"forwards":1},"description":"Connects a GitHub post to an entity that it references.\n\nFor example, if you write a GitHub issue comment that says \"thanks\n@username for pull #1337\", it will create references edges to both the user\n@username, and to pull #1337 in the same repository.\n","forwardName":"references","prefix":"E\u0000sourcecred\u0000github\u0000REFERENCES\u0000"},{"backwardName":"got 👍 from","defaultWeight":{"backwards":0,"forwards":1},"description":"Connects users to posts to which they gave a 👍 reaction.\n","forwardName":"reacted 👍 to","prefix":"E\u0000sourcecred\u0000github\u0000REACTS\u0000THUMBS_UP\u0000"},{"backwardName":"got ❤️ from","defaultWeight":{"backwards":0,"forwards":2},"description":"Connects users to posts to which they gave a ❤️ reaction.\n","forwardName":"reacted ❤️ to","prefix":"E\u0000sourcecred\u0000github\u0000REACTS\u0000HEART\u0000"},{"backwardName":"got 🎉 from","defaultWeight":{"backwards":0,"forwards":4},"description":"Connects users to posts to which they gave a 🎉 reaction.\n","forwardName":"reacted 🎉 to","prefix":"E\u0000sourcecred\u0000github\u0000REACTS\u0000HOORAY\u0000"},{"backwardName":"got 🚀 from","defaultWeight":{"backwards":0,"forwards":1},"description":"Connects users to posts to which they gave a 🚀 reaction.\n","forwardName":"reacted 🚀 to","prefix":"E\u0000sourcecred\u0000github\u0000REACTS\u0000ROCKET\u0000"},{"backwardName":"merged on GitHub as","defaultWeight":{"backwards":1,"forwards":1},"description":"Connects a commit on GitHub to the corresponding raw Git commit.\n","forwardName":"corresponds to Git commit","prefix":"E\u0000sourcecred\u0000github\u0000CORRESPONDS_TO_COMMIT_TYPE\u0000"}],"name":"GitHub","nodePrefix":"N\u0000sourcecred\u0000github\u0000","nodeTypes":[{"defaultWeight":4,"description":"NodeType for a GitHub repository","name":"Repository","pluralName":"Repositories","prefix":"N\u0000sourcecred\u0000github\u0000REPO\u0000"},{"defaultWeight":2,"description":"NodeType for a GitHub issue","name":"Issue","pluralName":"Issues","prefix":"N\u0000sourcecred\u0000github\u0000ISSUE\u0000"},{"defaultWeight":4,"description":"NodeType for a GitHub pull request","name":"Pull request","pluralName":"Pull requests","prefix":"N\u0000sourcecred\u0000github\u0000PULL\u0000"},{"defaultWeight":1,"description":"NodeType for a GitHub code review","name":"Pull request review","pluralName":"Pull request reviews","prefix":"N\u0000sourcecred\u0000github\u0000REVIEW\u0000"},{"defaultWeight":1,"description":"NodeType for a GitHub comment","name":"Comment","pluralName":"Comments","prefix":"N\u0000sourcecred\u0000github\u0000COMMENT\u0000"},{"defaultWeight":1,"description":"Represents a particular Git commit on GitHub, i.e. scoped to a particular repository","name":"Commit","pluralName":"Commits","prefix":"N\u0000sourcecred\u0000github\u0000COMMIT\u0000"},{"defaultWeight":0,"description":"NodeType for a GitHub user","name":"User","pluralName":"Users","prefix":"N\u0000sourcecred\u0000github\u0000USERLIKE\u0000USER\u0000"},{"defaultWeight":0,"description":"NodeType for a GitHub bot account","name":"Bot","pluralName":"Bots","prefix":"N\u0000sourcecred\u0000github\u0000USERLIKE\u0000BOT\u0000"}],"userTypes":[{"defaultWeight":0,"description":"NodeType for a GitHub user","name":"User","pluralName":"Users","prefix":"N\u0000sourcecred\u0000github\u0000USERLIKE\u0000USER\u0000"}]}]] \ No newline at end of file diff --git a/src/api/load.js b/src/api/load.js index 7a66b8a..db1c1b0 100644 --- a/src/api/load.js +++ b/src/api/load.js @@ -12,7 +12,10 @@ import {type TimelineCredParameters} from "../analysis/timeline/params"; import {type Project} from "../core/project"; import {setupProjectDirectory} from "../core/project_io"; -import {type PluginDeclaration} from "../analysis/pluginDeclaration"; +import { + type PluginDeclaration, + toJSON as pluginsToJSON, +} from "../analysis/pluginDeclaration"; import * as Discourse from "../plugins/discourse/loadWeightedGraph"; import * as Github from "../plugins/github/loadWeightedGraph"; import * as WeightedGraph from "../core/weightedGraph"; @@ -107,6 +110,10 @@ export async function load( const graphJSON = WeightedGraph.toJSON(weightedGraph); await fs.writeFile(graphFile, stringify(graphJSON)); + const pluginsFile = path.join(projectDirectory, "pluginDeclarations.json"); + const pluginsJSON = pluginsToJSON(plugins); + await fs.writeFile(pluginsFile, stringify(pluginsJSON)); + taskReporter.start("compute-cred"); const cred = await TimelineCred.compute({ weightedGraph, diff --git a/src/api/load.test.js b/src/api/load.test.js index 58848d3..472508b 100644 --- a/src/api/load.test.js +++ b/src/api/load.test.js @@ -27,6 +27,7 @@ import { } from "../analysis/timeline/params"; import * as WeightedGraph from "../core/weightedGraph"; import {DataDirectory} from "../backend/dataDirectory"; +import {fromJSON as pluginsFromJSON} from "../analysis/pluginDeclaration"; type JestMockFn = $Call; jest.mock("../plugins/github/loadWeightedGraph", () => ({ @@ -236,4 +237,17 @@ describe("api/load", () => { const expectedJSON = WeightedGraph.toJSON(identityGraph); expect(graphJSON).toEqual(expectedJSON); }); + + it("saves plugin declarations to disk", async () => { + const {options, taskReporter, sourcecredDirectory} = example(); + await load(options, taskReporter); + const projectDirectory = directoryForProjectId( + project.id, + sourcecredDirectory + ); + const pluginsFile = path.join(projectDirectory, "pluginDeclarations.json"); + const pluginsJSON = JSON.parse(await fs.readFile(pluginsFile)); + const actualPlugins = pluginsFromJSON(pluginsJSON); + expect(actualPlugins).toEqual(plugins); + }); }); diff --git a/src/backend/dataDirectory.js b/src/backend/dataDirectory.js index d1ea2b4..e9909c7 100644 --- a/src/backend/dataDirectory.js +++ b/src/backend/dataDirectory.js @@ -12,6 +12,7 @@ import type { ProjectStorageProvider, ProjectStorageExtras, } from "./projectStorage"; +import {toJSON as pluginsToJSON} from "../analysis/pluginDeclaration"; /** * Represents a SourceCred data directory. @@ -33,7 +34,7 @@ export class DataDirectory implements CacheProvider, ProjectStorageProvider { async storeProject( project: Project, - {weightedGraph, cred}: ProjectStorageExtras + {weightedGraph, cred, pluginDeclarations}: ProjectStorageExtras ): Promise { const projectDirectory = directoryForProjectId( project.id, @@ -45,11 +46,20 @@ export class DataDirectory implements CacheProvider, ProjectStorageProvider { await fs.writeFile(fileName, data); }; writeFile("project.json", stringify(projectToJSON(project))); - if (weightedGraph) + if (weightedGraph) { writeFile( "weightedGraph.json", stringify(WeightedGraph.toJSON(weightedGraph)) ); - if (cred) writeFile("cred.json", stringify(cred.toJSON())); + } + if (cred) { + writeFile("cred.json", stringify(cred.toJSON())); + } + if (pluginDeclarations) { + writeFile( + "pluginDeclarations.json", + stringify(pluginsToJSON(pluginDeclarations)) + ); + } } } diff --git a/src/backend/dataDirectory.test.js b/src/backend/dataDirectory.test.js index 65a4323..6a9b389 100644 --- a/src/backend/dataDirectory.test.js +++ b/src/backend/dataDirectory.test.js @@ -10,10 +10,12 @@ import {type CacheProvider} from "./cache"; import {type ProjectStorageProvider} from "./projectStorage"; import {DataDirectory} from "./dataDirectory"; import * as WeightedGraph from "../core/weightedGraph"; +import {toJSON as pluginsToJSON} from "../analysis/pluginDeclaration"; const project = createProject({id: "testing-project"}); const fakeWeightedGraph = deepFreeze(WeightedGraph.empty()); +const fakeDeclarations = deepFreeze([]); const fakeCred = ({ toJSON: () => ({is: "fake-cred"}), @@ -22,6 +24,7 @@ const fakeCred = ({ const fakeExtras = { weightedGraph: fakeWeightedGraph, cred: fakeCred, + pluginDeclarations: fakeDeclarations, }; describe("src/backend/dataDirectory", () => { @@ -111,6 +114,10 @@ describe("src/backend/dataDirectory", () => { WeightedGraph.toJSON(fakeWeightedGraph) ); await expectJSONFile("cred.json", fakeCred.toJSON()); + await expectJSONFile( + "pluginDeclarations.json", + pluginsToJSON(fakeDeclarations) + ); }); it("should work when sourcecredDirectory doesn't exist", async () => { @@ -141,6 +148,10 @@ describe("src/backend/dataDirectory", () => { WeightedGraph.toJSON(fakeWeightedGraph) ); await expectJSONFile("cred.json", fakeCred.toJSON()); + await expectJSONFile( + "pluginDeclarations.json", + pluginsToJSON(fakeDeclarations) + ); }); it("should fail when sourcecredDirectory is a file", async () => { diff --git a/src/backend/projectStorage.js b/src/backend/projectStorage.js index 78480d9..7aecafa 100644 --- a/src/backend/projectStorage.js +++ b/src/backend/projectStorage.js @@ -3,10 +3,12 @@ import {type WeightedGraph} from "../core/weightedGraph"; import {TimelineCred} from "../analysis/timeline/timelineCred"; import {type Project} from "../core/project"; +import {type PluginDeclarations} from "../analysis/pluginDeclaration"; export type ProjectStorageExtras = { +weightedGraph?: WeightedGraph, +cred?: TimelineCred, + +pluginDeclarations?: PluginDeclarations, }; export interface ProjectStorageProvider {