Expose a ReferenceDetector to createPluginGraphs (#1673)

Eventually all plugins are expected to use the ReferenceDetector. This
commit composes the Github and Discourse detectors we can create, for now
exposing it as unused to `createPluginGraphs`.

src/backend/loadContext.js

@@ -68,6 +68,7 @@ export class LoadContext {
   +_declarations = PluginLoaders.declarations;
   +_updateMirror = PluginLoaders.updateMirror;
   +_createPluginGraphs = PluginLoaders.createPluginGraphs;
+  +_createReferenceDetector = PluginLoaders.createReferenceDetector;
   +_contractPluginGraphs = PluginLoaders.contractPluginGraphs;
   +_overrideWeights = WeightedGraph.overrideWeights;
   +_computeTask = ComputeFunction.computeTask;
@@ -98,11 +99,17 @@ export class LoadContext {
       this._options,
       project
     );
-    const pluginGraphs = await this._createPluginGraphs(
+    const referenceDetector = await this._createReferenceDetector(
       this._pluginLoaders,
       this._options,
       cachedProject
     );
+    const pluginGraphs = await this._createPluginGraphs(
+      this._pluginLoaders,
+      this._options,
+      cachedProject,
+      referenceDetector
+    );
     const contractedGraph = await this._contractPluginGraphs(
       this._pluginLoaders,
       pluginGraphs

src/backend/loadContext.test.js

@@ -9,6 +9,7 @@ import {LoadContext} from "./loadContext";
 
 const fakes = {
   declarations: ({fake: "declarations"}: any),
+  referenceDetector: ({fake: "referenceDetector"}: any),
   pluginGraphs: ({fake: "pluginGraphs"}: any),
   contractedGraph: ({fake: "contractedGraph"}: any),
   weightedGraph: ({fake: "weightedGraph"}: any),
@@ -44,6 +45,10 @@ const mockProxyMethods = (
       .proxyMethod("updateMirror")
       .mockResolvedValueOnce({project, cache}),
 
+    createReferenceDetector: spyBuilder
+      .proxyMethod("createReferenceDetector")
+      .mockResolvedValueOnce(fakes.referenceDetector),
+
     createPluginGraphs: spyBuilder
       .proxyMethod("createPluginGraphs")
       .mockResolvedValueOnce(fakes.pluginGraphs),
@@ -98,6 +103,7 @@ describe("src/backend/loadContext", () => {
           // Methods
           _declarations: expect.anything(),
           _updateMirror: expect.anything(),
+          _createReferenceDetector: expect.anything(),
           _createPluginGraphs: expect.anything(),
           _contractPluginGraphs: expect.anything(),
           _overrideWeights: expect.anything(),
@@ -140,11 +146,17 @@ describe("src/backend/loadContext", () => {
           expectedEnv,
           project
         );
-        expect(spies.createPluginGraphs).toBeCalledWith(
+        expect(spies.createReferenceDetector).toBeCalledWith(
           loadContext._pluginLoaders,
           expectedEnv,
           cachedProject
         );
+        expect(spies.createPluginGraphs).toBeCalledWith(
+          loadContext._pluginLoaders,
+          expectedEnv,
+          cachedProject,
+          fakes.referenceDetector
+        );
         expect(spies.contractPluginGraphs).toBeCalledWith(
           loadContext._pluginLoaders,
           fakes.pluginGraphs

src/backend/pluginLoaders.js

@@ -10,6 +10,10 @@ import {type GithubToken} from "../plugins/github/token";
 import {type Loader as GithubLoader} from "../plugins/github/loader";
 import {type Loader as IdentityLoader} from "../plugins/identity/loader";
 import {type Loader as DiscourseLoader} from "../plugins/discourse/loader";
+import {
+  type ReferenceDetector,
+  CascadingReferenceDetector,
+} from "../core/references";
 
 /**
  * A type combining all known plugin Loader interfaces.
@@ -97,13 +101,41 @@ export async function updateMirror(
   return {project, cache};
 }
 
+/**
+ * Creates a ReferenceDetector composing all plugin reference detectors
+ * requested by the project.
+ */
+export async function createReferenceDetector(
+  {github, discourse}: $Shape<PluginLoaders>,
+  {githubToken}: GraphEnv,
+  {cache, project}: CachedProject
+): Promise<ReferenceDetector> {
+  const refs: ReferenceDetector[] = [];
+  if (project.repoIds.length) {
+    // TODO: similar to create graph, rather not depend on the token (#1580).
+    if (!githubToken) {
+      throw new Error("Tried to load GitHub, but no GitHub token set");
+    }
+    refs.push(
+      await github.referenceDetector(project.repoIds, githubToken, cache)
+    );
+  }
+  if (project.discourseServer) {
+    refs.push(
+      await discourse.referenceDetector(project.discourseServer, cache)
+    );
+  }
+  return new CascadingReferenceDetector(refs);
+}
+
 /**
  * Creates PluginGraphs containing all plugins requested by the Project.
  */
 export async function createPluginGraphs(
   {github, discourse}: PluginLoaders,
   {githubToken}: GraphEnv,
-  {cache, project}: CachedProject
+  {cache, project}: CachedProject,
+  _unused_referenceDetector: ReferenceDetector
 ): Promise<PluginGraphs> {
   const tasks: Promise<WeightedGraphT>[] = [];
   if (project.discourseServer) {

src/backend/pluginLoaders.test.js

@@ -1,6 +1,10 @@
 // @flow
 
 import {type CacheProvider} from "./cache";
+import {
+  type ReferenceDetector,
+  CascadingReferenceDetector,
+} from "../core/references";
 import * as WeightedGraph from "../core/weightedGraph";
 import {node as graphNode} from "../core/graphTestUtil";
 import {createProject} from "../core/project";
@@ -23,7 +27,9 @@ const mockGraphs = {
 
 const fakes = {
   githubDeclaration: ({fake: "githubDeclaration"}: any),
+  githubReferences: ({fake: "githubReferences"}: any),
   discourseDeclaration: ({fake: "discourseDeclaration"}: any),
+  discourseReferences: ({fake: "discourseReferences"}: any),
   identityDeclaration: ({fake: "identityDeclaration"}: any),
 };
 
@@ -31,15 +37,21 @@ const mockCacheProvider = (): CacheProvider => ({
   database: jest.fn(),
 });
 
+const mockReferenceDetector = (): ReferenceDetector => ({
+  addressFromUrl: jest.fn(),
+});
+
 const mockPluginLoaders = () => ({
   github: {
     declaration: jest.fn().mockReturnValue(fakes.githubDeclaration),
     updateMirror: jest.fn(),
+    referenceDetector: jest.fn().mockResolvedValue(fakes.githubReferences),
     createGraph: jest.fn().mockResolvedValue(mockGraphs.github),
   },
   discourse: {
     declaration: jest.fn().mockReturnValue(fakes.discourseDeclaration),
     updateMirror: jest.fn(),
+    referenceDetector: jest.fn().mockResolvedValue(fakes.discourseReferences),
     createGraph: jest.fn().mockResolvedValue(mockGraphs.discourse),
   },
   identity: {
@@ -185,6 +197,7 @@ describe("src/backend/pluginLoaders", () => {
   describe("createPluginGraphs", () => {
     it("should create discourse graph", async () => {
       // Given
+      const references = mockReferenceDetector();
       const loaders = mockPluginLoaders();
       const cache = mockCacheProvider();
       const githubToken = null;
@@ -198,7 +211,8 @@ describe("src/backend/pluginLoaders", () => {
       const pluginGraphs = await PluginLoaders.createPluginGraphs(
         loaders,
         {githubToken},
-        cachedProject
+        cachedProject,
+        references
       );
 
       // Then
@@ -216,6 +230,7 @@ describe("src/backend/pluginLoaders", () => {
 
     it("fail when missing GithubToken", async () => {
       // Given
+      const references = mockReferenceDetector();
       const loaders = mockPluginLoaders();
       const cache = mockCacheProvider();
       const githubToken = null;
@@ -229,7 +244,8 @@ describe("src/backend/pluginLoaders", () => {
       const p = PluginLoaders.createPluginGraphs(
         loaders,
         {githubToken},
-        cachedProject
+        cachedProject,
+        references
       );
 
       // Then
@@ -240,6 +256,7 @@ describe("src/backend/pluginLoaders", () => {
 
     it("should create github graph", async () => {
       // Given
+      const references = mockReferenceDetector();
       const loaders = mockPluginLoaders();
       const cache = mockCacheProvider();
       const githubToken = exampleGithubToken;
@@ -253,7 +270,8 @@ describe("src/backend/pluginLoaders", () => {
       const pluginGraphs = await PluginLoaders.createPluginGraphs(
         loaders,
         {githubToken},
-        cachedProject
+        cachedProject,
+        references
       );
 
       // Then
@@ -271,6 +289,35 @@ describe("src/backend/pluginLoaders", () => {
     });
   });
 
+  describe("createReferenceDetector", () => {
+    it("should create a CascadingReferenceDetector", async () => {
+      // Given
+      const loaders = mockPluginLoaders();
+      const cache = mockCacheProvider();
+      const githubToken = exampleGithubToken;
+      const project = createProject({
+        id: "has-github-and-discourse",
+        discourseServer: {serverUrl: "http://foo.bar"},
+        repoIds: [exampleRepoId],
+      });
+      const cachedProject = ({project, cache}: any);
+
+      // When
+      const references = await PluginLoaders.createReferenceDetector(
+        loaders,
+        {githubToken},
+        cachedProject
+      );
+
+      // Then
+      expect(references).toBeInstanceOf(CascadingReferenceDetector);
+      expect(((references: any): CascadingReferenceDetector).refs).toEqual([
+        fakes.githubReferences,
+        fakes.discourseReferences,
+      ]);
+    });
+  });
+
   describe("contractPluginGraphs", () => {
     it("should only merge graphs when no identities are defined", async () => {
       // Given

src/plugins/discourse/loader.js

@@ -4,11 +4,13 @@ import base64url from "base64url";
 import {TaskReporter} from "../../util/taskReporter";
 import {type CacheProvider} from "../../backend/cache";
 import {type WeightedGraph} from "../../core/weightedGraph";
+import {type ReferenceDetector} from "../../core/references";
 import {type PluginDeclaration} from "../../analysis/pluginDeclaration";
 import {type DiscourseServer} from "./server";
 import {Mirror} from "./mirror";
 import {SqliteMirrorRepository} from "./mirrorRepository";
 import {weightsForDeclaration} from "../../analysis/pluginDeclaration";
+import {DiscourseReferenceDetector} from "./referenceDetector";
 import {createGraph as _createGraph} from "./createGraph";
 import {declaration} from "./declaration";
 import {Fetcher} from "./fetch";
@@ -20,6 +22,10 @@ export interface Loader {
     cache: CacheProvider,
     reporter: TaskReporter
   ): Promise<void>;
+  referenceDetector(
+    server: DiscourseServer,
+    cache: CacheProvider
+  ): Promise<ReferenceDetector>;
   createGraph(
     server: DiscourseServer,
     cache: CacheProvider
@@ -29,6 +35,7 @@ export interface Loader {
 export default ({
   declaration: () => declaration,
   updateMirror,
+  referenceDetector,
   createGraph,
 }: Loader);
 
@@ -44,6 +51,14 @@ export async function updateMirror(
   await mirror.update(reporter);
 }
 
+export async function referenceDetector(
+  {serverUrl}: DiscourseServer,
+  cache: CacheProvider
+): Promise<ReferenceDetector> {
+  const repo = await repository(cache, serverUrl);
+  return new DiscourseReferenceDetector(repo);
+}
+
 export async function createGraph(
   {serverUrl}: DiscourseServer,
   cache: CacheProvider

src/plugins/github/loader.js

@@ -3,6 +3,7 @@
 import {TaskReporter} from "../../util/taskReporter";
 import {type CacheProvider} from "../../backend/cache";
 import {type WeightedGraph} from "../../core/weightedGraph";
+import {type ReferenceDetector} from "../../core/references";
 import {type PluginDeclaration} from "../../analysis/pluginDeclaration";
 import {type GithubToken} from "./token";
 import {Graph} from "../../core/graph";
@@ -15,6 +16,7 @@ import {
   default as fetchGithubRepo,
   fetchGithubRepoFromCache,
 } from "./fetchGithubRepo";
+import {fromRelationalViews as referenceDetectorFromRelationalViews} from "./referenceDetector";
 
 export interface Loader {
   declaration(): PluginDeclaration;
@@ -24,6 +26,11 @@ export interface Loader {
     cache: CacheProvider,
     reporter: TaskReporter
   ): Promise<void>;
+  referenceDetector(
+    repoIds: $ReadOnlyArray<RepoId>,
+    token: GithubToken,
+    cache: CacheProvider
+  ): Promise<ReferenceDetector>;
   createGraph(
     repoIds: $ReadOnlyArray<RepoId>,
     token: GithubToken,
@@ -33,6 +40,7 @@ export interface Loader {
 
 export default ({
   declaration: () => declaration,
+  referenceDetector,
   updateMirror,
   createGraph,
 }: Loader);
@@ -54,6 +62,21 @@ export async function updateMirror(
   }
 }
 
+export async function referenceDetector(
+  repoIds: $ReadOnlyArray<RepoId>,
+  token: GithubToken,
+  cache: CacheProvider
+): Promise<ReferenceDetector> {
+  const rvs = [];
+  for (const repoId of repoIds) {
+    const repo = await fetchGithubRepoFromCache(repoId, {token, cache});
+    const rv = new RelationalView();
+    rv.addRepository(repo);
+    rvs.push(rv);
+  }
+  return referenceDetectorFromRelationalViews(rvs);
+}
+
 export async function createGraph(
   repoIds: $ReadOnlyArray<RepoId>,
   token: GithubToken,