Backend: implement PluginLoaders.updateMirror (#1617)

Note, the return type is a CachedProject. See #1586 for discussion.
Having this type allows us to create new functions with a semantic
of requiring the project is mirrored into cache. It is opaque,
because only the "all plugins" semantic which PluginsLoaders has
could know when mirroring of a Project has been completed.

Additionally MirrorEnv is not a strict type. We're expecting this
to be a subset of parameters. We'll use Flow to ensure we only use
the ones we need from it.

src/backend/pluginLoaders.js

@@ -1,7 +1,10 @@
 //@flow
 
+import {TaskReporter} from "../util/taskReporter";
 import {type Project} from "../core/project";
 import {type PluginDeclaration} from "../analysis/pluginDeclaration";
+import {type CacheProvider} from "./cache";
+import {type GithubToken} from "../plugins/github/token";
 import {type Loader as GithubLoader} from "../plugins/github/loader";
 import {type Loader as IdentityLoader} from "../plugins/identity/loader";
 import {type Loader as DiscourseLoader} from "../plugins/discourse/loader";
@@ -18,6 +21,22 @@ export type PluginLoaders = {|
   +identity: IdentityLoader,
 |};
 
+/**
+ * Represents a Project which has been mirrored into the CacheProvider.
+ *
+ * Note: no guarantees about the cache are made, it's state is a best effort.
+ */
+opaque type CachedProject = {|
+  +cache: CacheProvider,
+  +project: Project,
+|};
+
+type MirrorEnv = {
+  +githubToken: ?GithubToken,
+  +reporter: TaskReporter,
+  +cache: CacheProvider,
+};
+
 /**
  * Gets all relevant PluginDeclarations for a given Project.
  */
@@ -37,3 +56,29 @@ export function declarations(
   }
   return plugins;
 }
+
+/**
+ * Updates all mirrors into cache as requested by the Project.
+ */
+export async function updateMirror(
+  {github, discourse}: PluginLoaders,
+  {githubToken, cache, reporter}: MirrorEnv,
+  project: Project
+): Promise<CachedProject> {
+  const tasks: Promise<void>[] = [];
+  if (project.discourseServer) {
+    tasks.push(
+      discourse.updateMirror(project.discourseServer, cache, reporter)
+    );
+  }
+  if (project.repoIds.length) {
+    if (!githubToken) {
+      throw new Error("Tried to load GitHub, but no GitHub token set");
+    }
+    tasks.push(
+      github.updateMirror(project.repoIds, githubToken, cache, reporter)
+    );
+  }
+  await Promise.all(tasks);
+  return {project, cache};
+}

src/backend/pluginLoaders.test.js

@@ -1,9 +1,15 @@
 // @flow
 
 import {createProject} from "../core/project";
+import {TestTaskReporter} from "../util/taskReporter";
+import {validateToken} from "../plugins/github/token";
 import {makeRepoId} from "../plugins/github/repoId";
 import * as PluginLoaders from "./pluginLoaders";
 
+const mockCacheProvider = () => ({
+  database: jest.fn(),
+});
+
 const fakeGithubDec = ("fake-github-dec": any);
 const fakeDiscourseDec = ("fake-discourse-dec": any);
 const fakeIdentityDec = ("fake-identity-dec": any);
@@ -11,9 +17,11 @@ const fakeIdentityDec = ("fake-identity-dec": any);
 const mockPluginLoaders = () => ({
   github: {
     declaration: jest.fn().mockReturnValue(fakeGithubDec),
+    updateMirror: jest.fn(),
   },
   discourse: {
     declaration: jest.fn().mockReturnValue(fakeDiscourseDec),
+    updateMirror: jest.fn(),
   },
   identity: {
     declaration: jest.fn().mockReturnValue(fakeIdentityDec),
@@ -21,6 +29,7 @@ const mockPluginLoaders = () => ({
 });
 
 describe("src/backend/pluginLoaders", () => {
+  const exampleGithubToken = validateToken("0".repeat(40));
   const exampleRepoId = makeRepoId("sourcecred-test", "example-github");
 
   describe("declarations", () => {
@@ -69,4 +78,87 @@ describe("src/backend/pluginLoaders", () => {
       expect(decs).toEqual([fakeIdentityDec]);
     });
   });
+
+  describe("updateMirror", () => {
+    it("should update discourse mirror", async () => {
+      // Given
+      const loaders = mockPluginLoaders();
+      const cache = mockCacheProvider();
+      const reporter = new TestTaskReporter();
+      const githubToken = null;
+      const project = createProject({
+        id: "has-discourse",
+        discourseServer: {serverUrl: "http://foo.bar"},
+      });
+
+      // When
+      await PluginLoaders.updateMirror(
+        loaders,
+        {githubToken, cache, reporter},
+        project
+      );
+
+      // Then
+      const {discourse} = loaders;
+      expect(discourse.updateMirror).toBeCalledTimes(1);
+      expect(discourse.updateMirror).toBeCalledWith(
+        project.discourseServer,
+        cache,
+        reporter
+      );
+    });
+
+    it("should fail when missing GithubToken", async () => {
+      // Given
+      const loaders = mockPluginLoaders();
+      const cache = mockCacheProvider();
+      const githubToken = null;
+      const reporter = new TestTaskReporter();
+      const project = createProject({
+        id: "has-github",
+        repoIds: [exampleRepoId],
+      });
+
+      // When
+      const p = PluginLoaders.updateMirror(
+        loaders,
+        {githubToken, cache, reporter},
+        project
+      );
+
+      // Then
+      await expect(p).rejects.toThrow(
+        "Tried to load GitHub, but no GitHub token set"
+      );
+    });
+
+    it("should update github mirror", async () => {
+      // Given
+      const loaders = mockPluginLoaders();
+      const cache = mockCacheProvider();
+      const githubToken = exampleGithubToken;
+      const reporter = new TestTaskReporter();
+      const project = createProject({
+        id: "has-github",
+        repoIds: [exampleRepoId],
+      });
+
+      // When
+      await PluginLoaders.updateMirror(
+        loaders,
+        {githubToken, cache, reporter},
+        project
+      );
+
+      // Then
+      const {github} = loaders;
+      expect(github.updateMirror).toBeCalledTimes(1);
+      expect(github.updateMirror).toBeCalledWith(
+        project.repoIds,
+        githubToken,
+        cache,
+        reporter
+      );
+    });
+  });
 });

src/plugins/discourse/loader.js

@@ -1,12 +1,50 @@
 // @flow
 
+import base64url from "base64url";
+import {TaskReporter} from "../../util/taskReporter";
+import {type CacheProvider} from "../../backend/cache";
 import {type PluginDeclaration} from "../../analysis/pluginDeclaration";
+import {type MirrorOptions, Mirror} from "./mirror";
+import {SqliteMirrorRepository} from "./mirrorRepository";
 import {declaration} from "./declaration";
+import {Fetcher} from "./fetch";
+
+export type DiscourseServer = {|
+  +serverUrl: string,
+  +mirrorOptions?: $Shape<MirrorOptions>,
+|};
 
 export interface Loader {
   declaration(): PluginDeclaration;
+  updateMirror(
+    server: DiscourseServer,
+    cache: CacheProvider,
+    reporter: TaskReporter
+  ): Promise<void>;
 }
 
 export default ({
   declaration: () => declaration,
+  updateMirror,
 }: Loader);
+
+export async function updateMirror(
+  server: DiscourseServer,
+  cache: CacheProvider,
+  reporter: TaskReporter
+): Promise<void> {
+  const {serverUrl, mirrorOptions} = server;
+  const repo = await repository(cache, serverUrl);
+  const fetcher = new Fetcher({serverUrl});
+  const mirror = new Mirror(repo, fetcher, serverUrl, mirrorOptions);
+  await mirror.update(reporter);
+}
+
+async function repository(
+  cache: CacheProvider,
+  serverUrl: string
+): Promise<SqliteMirrorRepository> {
+  // TODO: should replace base64url with hex, to be case insensitive.
+  const db = await cache.database(base64url.encode(serverUrl));
+  return new SqliteMirrorRepository(db, serverUrl);
+}

src/plugins/github/loader.js

@@ -1,12 +1,41 @@
 // @flow
 
+import {TaskReporter} from "../../util/taskReporter";
+import {type CacheProvider} from "../../backend/cache";
 import {type PluginDeclaration} from "../../analysis/pluginDeclaration";
+import {type GithubToken} from "./token";
 import {declaration} from "./declaration";
+import {type RepoId, repoIdToString} from "./repoId";
+import {default as fetchGithubRepo} from "./fetchGithubRepo";
 
 export interface Loader {
   declaration(): PluginDeclaration;
+  updateMirror(
+    repoIds: $ReadOnlyArray<RepoId>,
+    token: GithubToken,
+    cache: CacheProvider,
+    reporter: TaskReporter
+  ): Promise<void>;
 }
 
 export default ({
   declaration: () => declaration,
+  updateMirror,
 }: Loader);
+
+export async function updateMirror(
+  repoIds: $ReadOnlyArray<RepoId>,
+  token: GithubToken,
+  cache: CacheProvider,
+  reporter: TaskReporter
+): Promise<void> {
+  for (const repoId of repoIds) {
+    const taskId = `github/${repoIdToString(repoId)}`;
+    reporter.start(taskId);
+    await fetchGithubRepo(repoId, {
+      token: token,
+      cache,
+    });
+    reporter.finish(taskId);
+  }
+}