api/load: save plugin declarations to disk (#1624)

This builds on #1623 and is another step towards separating cred
computation from plugin declarations, as described in #1557. Basically,
this will allow the frontend to get plugin declarations even if the
TimelineCred computation never saw them.

This commit modifies `api/load`, and adds a new facility to
`DataDirectory` for saving the PluginDeclarations (which will be used by
@Beanow's in-flight refactor of `api/load`).

Test plan: See included unit tests, also try loading a project and
inspect the newlys saved file.

sharness/__snapshots__/example-github-load/projects/c291cmNlY3JlZC10ZXN0L2V4YW1wbGUtZ2l0aHVi/pluginDeclarations.json

@@ -0,0 +1 @@
+[{"type":"sourcecred/pluginDeclarations","version":"0.1.0"},[{"edgePrefix":"E\u0000sourcecred\u0000github\u0000","edgeTypes":[{"backwardName":"is authored by","defaultWeight":{"backwards":1,"forwards":0.5},"description":"Connects a GitHub account to a post that they authored.\n\nExamples of posts include issues, pull requests, and comments.\n","forwardName":"authors","prefix":"E\u0000sourcecred\u0000github\u0000AUTHORS\u0000"},{"backwardName":"has child","defaultWeight":{"backwards":0.25,"forwards":1},"description":"Connects a GitHub entity to its child entities.\n\nFor example, a Repository has Issues and Pull Requests as children, and a\nPull Request has comments and reviews as children.\n","forwardName":"has parent","prefix":"E\u0000sourcecred\u0000github\u0000HAS_PARENT\u0000"},{"backwardName":"is merged by","defaultWeight":{"backwards":1,"forwards":0.5},"description":"Connects a GitHub pull request to the Git commit that it merges.\n","forwardName":"merges","prefix":"E\u0000sourcecred\u0000github\u0000MERGED_AS\u0000"},{"backwardName":"is referenced by","defaultWeight":{"backwards":0,"forwards":1},"description":"Connects a GitHub post to an entity that it references.\n\nFor example, if you write a GitHub issue comment that says \"thanks\n@username for pull #1337\", it will create references edges to both the user\n@username, and to pull #1337 in the same repository.\n","forwardName":"references","prefix":"E\u0000sourcecred\u0000github\u0000REFERENCES\u0000"},{"backwardName":"got 👍 from","defaultWeight":{"backwards":0,"forwards":1},"description":"Connects users to posts to which they gave a 👍 reaction.\n","forwardName":"reacted 👍 to","prefix":"E\u0000sourcecred\u0000github\u0000REACTS\u0000THUMBS_UP\u0000"},{"backwardName":"got ❤️ from","defaultWeight":{"backwards":0,"forwards":2},"description":"Connects users to posts to which they gave a ❤️ reaction.\n","forwardName":"reacted ❤️ to","prefix":"E\u0000sourcecred\u0000github\u0000REACTS\u0000HEART\u0000"},{"backwardName":"got 🎉 from","defaultWeight":{"backwards":0,"forwards":4},"description":"Connects users to posts to which they gave a 🎉 reaction.\n","forwardName":"reacted 🎉 to","prefix":"E\u0000sourcecred\u0000github\u0000REACTS\u0000HOORAY\u0000"},{"backwardName":"got 🚀 from","defaultWeight":{"backwards":0,"forwards":1},"description":"Connects users to posts to which they gave a 🚀 reaction.\n","forwardName":"reacted 🚀 to","prefix":"E\u0000sourcecred\u0000github\u0000REACTS\u0000ROCKET\u0000"},{"backwardName":"merged on GitHub as","defaultWeight":{"backwards":1,"forwards":1},"description":"Connects a commit on GitHub to the corresponding raw Git commit.\n","forwardName":"corresponds to Git commit","prefix":"E\u0000sourcecred\u0000github\u0000CORRESPONDS_TO_COMMIT_TYPE\u0000"}],"name":"GitHub","nodePrefix":"N\u0000sourcecred\u0000github\u0000","nodeTypes":[{"defaultWeight":4,"description":"NodeType for a GitHub repository","name":"Repository","pluralName":"Repositories","prefix":"N\u0000sourcecred\u0000github\u0000REPO\u0000"},{"defaultWeight":2,"description":"NodeType for a GitHub issue","name":"Issue","pluralName":"Issues","prefix":"N\u0000sourcecred\u0000github\u0000ISSUE\u0000"},{"defaultWeight":4,"description":"NodeType for a GitHub pull request","name":"Pull request","pluralName":"Pull requests","prefix":"N\u0000sourcecred\u0000github\u0000PULL\u0000"},{"defaultWeight":1,"description":"NodeType for a GitHub code review","name":"Pull request review","pluralName":"Pull request reviews","prefix":"N\u0000sourcecred\u0000github\u0000REVIEW\u0000"},{"defaultWeight":1,"description":"NodeType for a GitHub comment","name":"Comment","pluralName":"Comments","prefix":"N\u0000sourcecred\u0000github\u0000COMMENT\u0000"},{"defaultWeight":1,"description":"Represents a particular Git commit on GitHub, i.e. scoped to a particular repository","name":"Commit","pluralName":"Commits","prefix":"N\u0000sourcecred\u0000github\u0000COMMIT\u0000"},{"defaultWeight":0,"description":"NodeType for a GitHub user","name":"User","pluralName":"Users","prefix":"N\u0000sourcecred\u0000github\u0000USERLIKE\u0000USER\u0000"},{"defaultWeight":0,"description":"NodeType for a GitHub bot account","name":"Bot","pluralName":"Bots","prefix":"N\u0000sourcecred\u0000github\u0000USERLIKE\u0000BOT\u0000"}],"userTypes":[{"defaultWeight":0,"description":"NodeType for a GitHub user","name":"User","pluralName":"Users","prefix":"N\u0000sourcecred\u0000github\u0000USERLIKE\u0000USER\u0000"}]}]]

src/api/load.js

@@ -12,7 +12,10 @@ import {type TimelineCredParameters} from "../analysis/timeline/params";
 
 import {type Project} from "../core/project";
 import {setupProjectDirectory} from "../core/project_io";
-import {type PluginDeclaration} from "../analysis/pluginDeclaration";
+import {
+  type PluginDeclaration,
+  toJSON as pluginsToJSON,
+} from "../analysis/pluginDeclaration";
 import * as Discourse from "../plugins/discourse/loadWeightedGraph";
 import * as Github from "../plugins/github/loadWeightedGraph";
 import * as WeightedGraph from "../core/weightedGraph";
@@ -107,6 +110,10 @@ export async function load(
   const graphJSON = WeightedGraph.toJSON(weightedGraph);
   await fs.writeFile(graphFile, stringify(graphJSON));
 
+  const pluginsFile = path.join(projectDirectory, "pluginDeclarations.json");
+  const pluginsJSON = pluginsToJSON(plugins);
+  await fs.writeFile(pluginsFile, stringify(pluginsJSON));
+
   taskReporter.start("compute-cred");
   const cred = await TimelineCred.compute({
     weightedGraph,

src/api/load.test.js

@@ -27,6 +27,7 @@ import {
 } from "../analysis/timeline/params";
 import * as WeightedGraph from "../core/weightedGraph";
 import {DataDirectory} from "../backend/dataDirectory";
+import {fromJSON as pluginsFromJSON} from "../analysis/pluginDeclaration";
 
 type JestMockFn = $Call<typeof jest.fn>;
 jest.mock("../plugins/github/loadWeightedGraph", () => ({
@@ -236,4 +237,17 @@ describe("api/load", () => {
     const expectedJSON = WeightedGraph.toJSON(identityGraph);
     expect(graphJSON).toEqual(expectedJSON);
   });
+
+  it("saves plugin declarations to disk", async () => {
+    const {options, taskReporter, sourcecredDirectory} = example();
+    await load(options, taskReporter);
+    const projectDirectory = directoryForProjectId(
+      project.id,
+      sourcecredDirectory
+    );
+    const pluginsFile = path.join(projectDirectory, "pluginDeclarations.json");
+    const pluginsJSON = JSON.parse(await fs.readFile(pluginsFile));
+    const actualPlugins = pluginsFromJSON(pluginsJSON);
+    expect(actualPlugins).toEqual(plugins);
+  });
 });

src/backend/dataDirectory.js

@@ -12,6 +12,7 @@ import type {
   ProjectStorageProvider,
   ProjectStorageExtras,
 } from "./projectStorage";
+import {toJSON as pluginsToJSON} from "../analysis/pluginDeclaration";
 
 /**
  * Represents a SourceCred data directory.
@@ -33,7 +34,7 @@ export class DataDirectory implements CacheProvider, ProjectStorageProvider {
 
   async storeProject(
     project: Project,
-    {weightedGraph, cred}: ProjectStorageExtras
+    {weightedGraph, cred, pluginDeclarations}: ProjectStorageExtras
   ): Promise<void> {
     const projectDirectory = directoryForProjectId(
       project.id,
@@ -45,11 +46,20 @@ export class DataDirectory implements CacheProvider, ProjectStorageProvider {
       await fs.writeFile(fileName, data);
     };
     writeFile("project.json", stringify(projectToJSON(project)));
-    if (weightedGraph)
+    if (weightedGraph) {
       writeFile(
         "weightedGraph.json",
         stringify(WeightedGraph.toJSON(weightedGraph))
       );
-    if (cred) writeFile("cred.json", stringify(cred.toJSON()));
+    }
+    if (cred) {
+      writeFile("cred.json", stringify(cred.toJSON()));
+    }
+    if (pluginDeclarations) {
+      writeFile(
+        "pluginDeclarations.json",
+        stringify(pluginsToJSON(pluginDeclarations))
+      );
+    }
   }
 }

src/backend/dataDirectory.test.js

@@ -10,10 +10,12 @@ import {type CacheProvider} from "./cache";
 import {type ProjectStorageProvider} from "./projectStorage";
 import {DataDirectory} from "./dataDirectory";
 import * as WeightedGraph from "../core/weightedGraph";
+import {toJSON as pluginsToJSON} from "../analysis/pluginDeclaration";
 
 const project = createProject({id: "testing-project"});
 
 const fakeWeightedGraph = deepFreeze(WeightedGraph.empty());
+const fakeDeclarations = deepFreeze([]);
 
 const fakeCred = ({
   toJSON: () => ({is: "fake-cred"}),
@@ -22,6 +24,7 @@ const fakeCred = ({
 const fakeExtras = {
   weightedGraph: fakeWeightedGraph,
   cred: fakeCred,
+  pluginDeclarations: fakeDeclarations,
 };
 
 describe("src/backend/dataDirectory", () => {
@@ -111,6 +114,10 @@ describe("src/backend/dataDirectory", () => {
           WeightedGraph.toJSON(fakeWeightedGraph)
         );
         await expectJSONFile("cred.json", fakeCred.toJSON());
+        await expectJSONFile(
+          "pluginDeclarations.json",
+          pluginsToJSON(fakeDeclarations)
+        );
       });
 
       it("should work when sourcecredDirectory doesn't exist", async () => {
@@ -141,6 +148,10 @@ describe("src/backend/dataDirectory", () => {
           WeightedGraph.toJSON(fakeWeightedGraph)
         );
         await expectJSONFile("cred.json", fakeCred.toJSON());
+        await expectJSONFile(
+          "pluginDeclarations.json",
+          pluginsToJSON(fakeDeclarations)
+        );
       });
 
       it("should fail when sourcecredDirectory is a file", async () => {

src/backend/projectStorage.js

@@ -3,10 +3,12 @@
 import {type WeightedGraph} from "../core/weightedGraph";
 import {TimelineCred} from "../analysis/timeline/timelineCred";
 import {type Project} from "../core/project";
+import {type PluginDeclarations} from "../analysis/pluginDeclaration";
 
 export type ProjectStorageExtras = {
   +weightedGraph?: WeightedGraph,
   +cred?: TimelineCred,
+  +pluginDeclarations?: PluginDeclarations,
 };
 
 export interface ProjectStorageProvider {