Backend: implement PluginLoaders.createPluginGraphs (#1618)

Similar to CachedProject, we're using an opaque PluginGraphs return
type. Because only PluginLoaders can add the semantic of this being
all plugin graphs, and to use this semantic in future functions.

src/backend/pluginLoaders.js

@@ -2,6 +2,7 @@
 
 import {TaskReporter} from "../util/taskReporter";
 import {type Project} from "../core/project";
+import {type WeightedGraph as WeightedGraphT} from "../core/weightedGraph";
 import {type PluginDeclaration} from "../analysis/pluginDeclaration";
 import {type CacheProvider} from "./cache";
 import {type GithubToken} from "../plugins/github/token";
@@ -31,12 +32,24 @@ opaque type CachedProject = {|
   +project: Project,
 |};
 
+/**
+ * Represents all disjoint WeightedGraphs for a CachedProject.
+ */
+opaque type PluginGraphs = {|
+  +graphs: $ReadOnlyArray<WeightedGraphT>,
+  +cachedProject: CachedProject,
+|};
+
 type MirrorEnv = {
   +githubToken: ?GithubToken,
   +reporter: TaskReporter,
   +cache: CacheProvider,
 };
 
+type GraphEnv = {
+  +githubToken: ?GithubToken,
+};
+
 /**
  * Gets all relevant PluginDeclarations for a given Project.
  */
@@ -82,3 +95,31 @@ export async function updateMirror(
   await Promise.all(tasks);
   return {project, cache};
 }
+
+/**
+ * Creates PluginGraphs containing all plugins requested by the Project.
+ */
+export async function createPluginGraphs(
+  {github, discourse}: PluginLoaders,
+  {githubToken}: GraphEnv,
+  {cache, project}: CachedProject
+): Promise<PluginGraphs> {
+  const tasks: Promise<WeightedGraphT>[] = [];
+  if (project.discourseServer) {
+    tasks.push(discourse.createGraph(project.discourseServer, cache));
+  }
+  if (project.repoIds.length) {
+    if (!githubToken) {
+      throw new Error("Tried to load GitHub, but no GitHub token set");
+    }
+    tasks.push(github.createGraph(project.repoIds, githubToken, cache));
+  }
+
+  // It's important to use Promise.all so that we can load the plugins in
+  // parallel -- since loading is often IO-bound, this can be a big performance
+  // improvement.
+  return {
+    graphs: await Promise.all(tasks),
+    cachedProject: {cache, project},
+  };
+}

src/backend/pluginLoaders.test.js

@@ -1,11 +1,28 @@
 // @flow
 
+import * as WeightedGraph from "../core/weightedGraph";
+import {node as graphNode} from "../core/graphTestUtil";
 import {createProject} from "../core/project";
 import {TestTaskReporter} from "../util/taskReporter";
 import {validateToken} from "../plugins/github/token";
 import {makeRepoId} from "../plugins/github/repoId";
 import * as PluginLoaders from "./pluginLoaders";
 
+const githubSentinel = graphNode("github-sentinel");
+const discourseSentinel = graphNode("discourse-sentinel");
+
+const mockGithubGraph = () => {
+  const wg = WeightedGraph.empty();
+  wg.graph.addNode(githubSentinel);
+  return wg;
+};
+
+const mockDiscourseGraph = () => {
+  const wg = WeightedGraph.empty();
+  wg.graph.addNode(discourseSentinel);
+  return wg;
+};
+
 const mockCacheProvider = () => ({
   database: jest.fn(),
 });
@@ -18,10 +35,12 @@ const mockPluginLoaders = () => ({
   github: {
     declaration: jest.fn().mockReturnValue(fakeGithubDec),
     updateMirror: jest.fn(),
+    createGraph: jest.fn().mockResolvedValue(mockGithubGraph()),
   },
   discourse: {
     declaration: jest.fn().mockReturnValue(fakeDiscourseDec),
     updateMirror: jest.fn(),
+    createGraph: jest.fn().mockResolvedValue(mockDiscourseGraph()),
   },
   identity: {
     declaration: jest.fn().mockReturnValue(fakeIdentityDec),
@@ -161,4 +180,93 @@ describe("src/backend/pluginLoaders", () => {
       );
     });
   });
+
+  describe("createPluginGraphs", () => {
+    it("should create discourse graph", async () => {
+      // Given
+      const loaders = mockPluginLoaders();
+      const cache = mockCacheProvider();
+      const githubToken = null;
+      const project = createProject({
+        id: "has-discourse",
+        discourseServer: {serverUrl: "http://foo.bar"},
+      });
+      const cachedProject = ({project, cache}: any);
+
+      // When
+      const pluginGraphs = await PluginLoaders.createPluginGraphs(
+        loaders,
+        {githubToken},
+        cachedProject
+      );
+
+      // Then
+      const {discourse} = loaders;
+      expect(pluginGraphs).toEqual({
+        graphs: [mockDiscourseGraph()],
+        cachedProject,
+      });
+      expect(discourse.createGraph).toBeCalledTimes(1);
+      expect(discourse.createGraph).toBeCalledWith(
+        project.discourseServer,
+        cache
+      );
+    });
+
+    it("fail when missing GithubToken", async () => {
+      // Given
+      const loaders = mockPluginLoaders();
+      const cache = mockCacheProvider();
+      const githubToken = null;
+      const project = createProject({
+        id: "has-github",
+        repoIds: [exampleRepoId],
+      });
+      const cachedProject = ({project, cache}: any);
+
+      // When
+      const p = PluginLoaders.createPluginGraphs(
+        loaders,
+        {githubToken},
+        cachedProject
+      );
+
+      // Then
+      await expect(p).rejects.toThrow(
+        "Tried to load GitHub, but no GitHub token set"
+      );
+    });
+
+    it("should create github graph", async () => {
+      // Given
+      const loaders = mockPluginLoaders();
+      const cache = mockCacheProvider();
+      const githubToken = exampleGithubToken;
+      const project = createProject({
+        id: "has-github",
+        repoIds: [exampleRepoId],
+      });
+      const cachedProject = ({project, cache}: any);
+
+      // When
+      const pluginGraphs = await PluginLoaders.createPluginGraphs(
+        loaders,
+        {githubToken},
+        cachedProject
+      );
+
+      // Then
+      const {github} = loaders;
+      expect(pluginGraphs).toEqual({
+        graphs: [mockGithubGraph()],
+        cachedProject,
+      });
+      expect(github.createGraph).toBeCalledTimes(1);
+      expect(github.createGraph).toBeCalledWith(
+        project.repoIds,
+        githubToken,
+        cache
+      );
+    });
+  });
 });

src/plugins/discourse/loader.js

@@ -3,9 +3,12 @@
 import base64url from "base64url";
 import {TaskReporter} from "../../util/taskReporter";
 import {type CacheProvider} from "../../backend/cache";
+import {type WeightedGraph} from "../../core/weightedGraph";
 import {type PluginDeclaration} from "../../analysis/pluginDeclaration";
 import {type MirrorOptions, Mirror} from "./mirror";
 import {SqliteMirrorRepository} from "./mirrorRepository";
+import {weightsForDeclaration} from "../../analysis/pluginDeclaration";
+import {createGraph as _createGraph} from "./createGraph";
 import {declaration} from "./declaration";
 import {Fetcher} from "./fetch";
 
@@ -21,11 +24,16 @@ export interface Loader {
     cache: CacheProvider,
     reporter: TaskReporter
   ): Promise<void>;
+  createGraph(
+    server: DiscourseServer,
+    cache: CacheProvider
+  ): Promise<WeightedGraph>;
 }
 
 export default ({
   declaration: () => declaration,
   updateMirror,
+  createGraph,
 }: Loader);
 
 export async function updateMirror(
@@ -40,6 +48,16 @@ export async function updateMirror(
   await mirror.update(reporter);
 }
 
+export async function createGraph(
+  {serverUrl}: DiscourseServer,
+  cache: CacheProvider
+): Promise<WeightedGraph> {
+  const repo = await repository(cache, serverUrl);
+  const graph = _createGraph(serverUrl, repo);
+  const weights = weightsForDeclaration(declaration);
+  return {graph, weights};
+}
+
 async function repository(
   cache: CacheProvider,
   serverUrl: string

src/plugins/github/loader.js

@@ -2,11 +2,19 @@
 
 import {TaskReporter} from "../../util/taskReporter";
 import {type CacheProvider} from "../../backend/cache";
+import {type WeightedGraph} from "../../core/weightedGraph";
 import {type PluginDeclaration} from "../../analysis/pluginDeclaration";
 import {type GithubToken} from "./token";
+import {Graph} from "../../core/graph";
 import {declaration} from "./declaration";
 import {type RepoId, repoIdToString} from "./repoId";
-import {default as fetchGithubRepo} from "./fetchGithubRepo";
+import {createGraph as _createGraph} from "./createGraph";
+import {RelationalView} from "./relationalView";
+import {weightsForDeclaration} from "../../analysis/pluginDeclaration";
+import {
+  default as fetchGithubRepo,
+  fetchGithubRepoFromCache,
+} from "./fetchGithubRepo";
 
 export interface Loader {
   declaration(): PluginDeclaration;
@@ -16,11 +24,17 @@ export interface Loader {
     cache: CacheProvider,
     reporter: TaskReporter
   ): Promise<void>;
+  createGraph(
+    repoIds: $ReadOnlyArray<RepoId>,
+    token: GithubToken,
+    cache: CacheProvider
+  ): Promise<WeightedGraph>;
 }
 
 export default ({
   declaration: () => declaration,
   updateMirror,
+  createGraph,
 }: Loader);
 
 export async function updateMirror(
@@ -39,3 +53,23 @@ export async function updateMirror(
     reporter.finish(taskId);
   }
 }
+
+export async function createGraph(
+  repoIds: $ReadOnlyArray<RepoId>,
+  token: GithubToken,
+  cache: CacheProvider
+): Promise<WeightedGraph> {
+  const repositories = [];
+  for (const repoId of repoIds) {
+    repositories.push(await fetchGithubRepoFromCache(repoId, {token, cache}));
+  }
+  const graph = Graph.merge(
+    repositories.map((r) => {
+      const rv = new RelationalView();
+      rv.addRepository(r);
+      return _createGraph(rv);
+    })
+  );
+  const weights = weightsForDeclaration(declaration);
+  return {graph, weights};
+}