Add core/project and core/project_io

This creates a new `Project` type which will replace `RepoId` as the
index type for saving and loading data.

The basic data type is added to `project.js`. Rather than having a
`RepoIdRegistry`, I intend to infer the registry at build time by
scanning for available projects saved in the sourcecred directory. I've
added the `project_io` module for this task. It has methods for setting
up a project subdirectory, and loading the `Project` info from that
subdirectory.

To ensure that projects ids can be encoded even if they have symbols
like `/` and `@`, we base64 encode them.

To ensure that project ids can be retrieved at build time, the
`getProjectIds` method is factored out into its own plain ECMAScript
module. For all non-build time needs, it is re-exported from
`project_io`.

Test plan: Unit tests added; run `yarn test`.

package.json

@@ -4,6 +4,7 @@
   "private": true,
   "dependencies": {
     "aphrodite": "^2.1.0",
+    "base-64": "^0.1.0",
     "better-sqlite3": "^5.4.0",
     "chalk": "2.4.2",
     "commonmark": "^0.29.0",

src/core/_getProjectIds.js

@@ -0,0 +1,36 @@
+// @flow
+
+// This file is a complement to `./project_io.js`. It contains the
+// implementation of `getProjectIds`, which is written in plain ECMAScript so
+// that we can depend on it at build time. Regular users should not depend on
+// this file; instead, depend on `./project_io.js`, which re-exports this
+// method.
+//
+// This file is tested in ./project_io.test.js
+
+const path = require("path");
+const base64 = require("base-64");
+const fs = require("fs-extra");
+
+module.exports = function getProjectIds(
+  sourcecredDirectory /*: string */
+) /*: $ReadOnlyArray<string> */ {
+  const projectsPath = path.join(sourcecredDirectory, "projects");
+  let entries = [];
+  try {
+    entries = fs.readdirSync(projectsPath);
+  } catch {
+    return [];
+  }
+  const projectIds = [];
+  for (const entry of entries) {
+    const jsonPath = path.join(projectsPath, entry, "project.json");
+    try {
+      fs.statSync(jsonPath);
+      projectIds.push(base64.decode(entry));
+    } catch {
+      continue;
+    }
+  }
+  return projectIds;
+};

src/core/project.js

@@ -0,0 +1,47 @@
+// @flow
+
+import base64 from "base-64";
+import {type RepoId} from "../core/repoId";
+import {toCompat, fromCompat, type Compatible} from "../util/compat";
+
+export type ProjectId = string;
+
+/**
+ * A project represents a scope for cred analysis.
+ *
+ * Right now it has an `id` (which should be unique across a user's projects)
+ * and an array of GitHub RepoIds.
+ *
+ * In the future, we will add support for more plugins (and remove the
+ * hardcoded GitHub support).
+ *
+ * We may add more fields (e.g. a description) to this object in the futre.
+ *
+ * We may create a complimentary object with load/cache info for the project in
+ * the future (e.g. showing the last update time for each of the project's data
+ * dependencies).
+ */
+export type Project = {|
+  +id: ProjectId,
+  +repoIds: $ReadOnlyArray<RepoId>,
+|};
+
+const COMPAT_INFO = {type: "sourcecred/project", version: "0.1.0"};
+
+export type ProjectJSON = Compatible<Project>;
+
+export function projectToJSON(p: Project): ProjectJSON {
+  return toCompat(COMPAT_INFO, p);
+}
+
+export function projectFromJSON(j: ProjectJSON): Project {
+  return fromCompat(COMPAT_INFO, j);
+}
+
+/**
+ * Encode the project ID so it can be stored on the filesystem,
+ * or retrieved via XHR from the frontend.
+ */
+export function encodeProjectId(id: ProjectId): string {
+  return base64.encode(id);
+}

src/core/project.test.js

@@ -0,0 +1,29 @@
+// @flow
+
+import {projectToJSON, projectFromJSON, type Project} from "./project";
+
+import {makeRepoId} from "./repoId";
+
+describe("core/project.js", () => {
+  const foobar = Object.freeze(makeRepoId("foo", "bar"));
+  const foozod = Object.freeze(makeRepoId("foo", "zod"));
+  const p1: Project = Object.freeze({
+    id: "foo/bar",
+    repoIds: Object.freeze([foobar]),
+  });
+  const p2: Project = Object.freeze({
+    id: "@foo",
+    repoIds: Object.freeze([foobar, foozod]),
+  });
+  describe("to/fro JSON", () => {
+    it("round trip is identity", () => {
+      function check(p: Project) {
+        const json = projectToJSON(p);
+        const p_ = projectFromJSON(json);
+        expect(p).toEqual(p_);
+      }
+      check(p1);
+      check(p2);
+    });
+  });
+});

src/core/project_io.js

@@ -0,0 +1,94 @@
+// @flow
+// This module contains logic for loading/saving projects to the
+// sourcecred directory.
+//
+// It is separated from project.js so that it's possible to depend on project
+// logic independent from anything related to the filesystem. (Depending
+// on the path or fs module in the frontend would create a build error.)
+
+import fs from "fs-extra";
+import path from "path";
+import stringify from "json-stable-stringify";
+
+import {
+  type Project,
+  type ProjectId,
+  projectToJSON,
+  projectFromJSON,
+  encodeProjectId,
+} from "./project";
+import _getProjectIds from "./_getProjectIds";
+
+/**
+ * Get the ids for every project saved on the filesystem.
+ *
+ * It is not guaranteed that it will be possible to load the id in question.
+ * (For example, the project may be malformed, or may have an outdated compat
+ * version.)
+ */
+export function getProjectIds(
+  sourcecredDirectory: string
+): $ReadOnlyArray<ProjectId> {
+  return _getProjectIds(sourcecredDirectory);
+}
+
+/**
+ * Returns the project directory for the given id.
+ *
+ * Does not guarantee that the project directory has been created;
+ * does not do any IO.
+ */
+export function directoryForProjectId(
+  id: ProjectId,
+  sourcecredDirectory: string
+): string {
+  return path.join(sourcecredDirectory, "projects", encodeProjectId(id));
+}
+
+/**
+ * Sets up a directory for the project, including a `project.json` file describing the project.
+ *
+ * If there is already a project.json file present, it will be over-written.
+ *
+ * Returns the project directory.
+ */
+export async function setupProjectDirectory(
+  project: Project,
+  sourcecredDirectory: string
+): Promise<string> {
+  const projectDirectory = directoryForProjectId(
+    project.id,
+    sourcecredDirectory
+  );
+  await fs.mkdirp(projectDirectory);
+  const projectFile = path.join(projectDirectory, "project.json");
+  await fs.writeFile(projectFile, stringify(projectToJSON(project)));
+  return projectDirectory;
+}
+
+/**
+ * Load the Project with given id from the sourcecred directory.
+ *
+ * This method may throw an error if the project is not present,
+ * or is malformed or corrupted.
+ */
+export async function loadProject(
+  id: ProjectId,
+  sourcecredDirectory: string
+): Promise<Project> {
+  const directory = directoryForProjectId(id, sourcecredDirectory);
+  const jsonPath = path.join(directory, "project.json");
+  try {
+    const contents = await fs.readFile(jsonPath);
+    const project: Project = projectFromJSON(JSON.parse(contents));
+    if (project.id !== id) {
+      throw new Error(`project ${project.id} saved under id ${id}`);
+    }
+    return project;
+  } catch (e) {
+    if (e.message.startsWith("ENOENT:")) {
+      throw `project ${id} not loaded`;
+    }
+    throw e;
+  }
+}

src/core/project_io.test.js

@@ -0,0 +1,129 @@
+// @flow
+
+import tmp from "tmp";
+import path from "path";
+import fs from "fs-extra";
+
+import {type Project, encodeProjectId, projectToJSON} from "./project";
+
+import {
+  getProjectIds,
+  setupProjectDirectory,
+  directoryForProjectId,
+  loadProject,
+} from "./project_io";
+
+import {makeRepoId} from "./repoId";
+
+describe("core/project_io.js", () => {
+  const foobar = Object.freeze(makeRepoId("foo", "bar"));
+  const foozod = Object.freeze(makeRepoId("foo", "zod"));
+  const p1: Project = Object.freeze({
+    id: "foo/bar",
+    repoIds: Object.freeze([foobar]),
+  });
+  const p2: Project = Object.freeze({
+    id: "@foo",
+    repoIds: Object.freeze([foobar, foozod]),
+  });
+
+  it("setupProjectDirectory results in a loadable project", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    await setupProjectDirectory(p1, sourcecredDirectory);
+    const ps = getProjectIds(sourcecredDirectory);
+    expect(ps).toEqual([p1.id]);
+    expect(await loadProject(p1.id, sourcecredDirectory)).toEqual(p1);
+  });
+  it("setupProjectDirectory twice results in two loadable projects", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    await setupProjectDirectory(p1, sourcecredDirectory);
+    await setupProjectDirectory(p2, sourcecredDirectory);
+    const ps = getProjectIds(sourcecredDirectory);
+    expect(ps).toHaveLength(2);
+    expect(ps.slice().sort()).toEqual([p2.id, p1.id]);
+    expect(await loadProject(p1.id, sourcecredDirectory)).toEqual(p1);
+    expect(await loadProject(p2.id, sourcecredDirectory)).toEqual(p2);
+  });
+  it("getProjectIds returns no projects if none were setup", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    const ps = getProjectIds(sourcecredDirectory);
+    expect(ps).toHaveLength(0);
+  });
+  it("setupProjectDirectory returns the right directory", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    const dir = await setupProjectDirectory(p1, sourcecredDirectory);
+    expect(dir).toEqual(directoryForProjectId(p1.id, sourcecredDirectory));
+    const projectJsonPath = path.join(dir, "project.json");
+    await fs.stat(projectJsonPath);
+  });
+  it("projects can be accessed using the encoded ID", async () => {
+    // Necessary so that frontend consumers can locate the project via the file mirror API
+    const sourcecredDirectory = tmp.dirSync().name;
+    await setupProjectDirectory(p1, sourcecredDirectory);
+    const projectJsonPath = path.join(
+      sourcecredDirectory,
+      "projects",
+      encodeProjectId(p1.id),
+      "project.json"
+    );
+    await fs.stat(projectJsonPath);
+  });
+  it("getProjectIds ignores non-project subdirectories", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    await setupProjectDirectory(p1, sourcecredDirectory);
+    await fs.mkdirp(path.join(sourcecredDirectory, "projects", "foobar"));
+    const ps = getProjectIds(sourcecredDirectory);
+    expect(ps).toEqual([p1.id]);
+  });
+  it("getProjectIds ignores non-project file entries", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    await setupProjectDirectory(p1, sourcecredDirectory);
+    fs.writeFileSync(
+      path.join(sourcecredDirectory, "projects", "foobar"),
+      "1234"
+    );
+    const ps = getProjectIds(sourcecredDirectory);
+    expect(ps).toEqual([p1.id]);
+  });
+  it("loadProject throws an error on inconsistent id", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    const projectDirectory = await setupProjectDirectory(
+      p1,
+      sourcecredDirectory
+    );
+    const jsonPath = path.join(projectDirectory, "project.json");
+    const badJson = projectToJSON(p2);
+    fs.writeFileSync(jsonPath, JSON.stringify(badJson));
+    expect.assertions(1);
+    return loadProject(p1.id, sourcecredDirectory).catch((e) =>
+      expect(e.message).toMatch(`project ${p2.id} saved under id ${p1.id}`)
+    );
+  });
+  it("loadProject throws an error on bad compat", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    const projectDirectory = await setupProjectDirectory(
+      p1,
+      sourcecredDirectory
+    );
+    const jsonPath = path.join(projectDirectory, "project.json");
+    const badJson = [{type: "sourcecred/project", version: "NaN"}, {}];
+    fs.writeFileSync(jsonPath, JSON.stringify(badJson));
+    expect.assertions(1);
+    return loadProject(p1.id, sourcecredDirectory).catch((e) =>
+      expect(e.message).toMatch(`tried to load unsupported version`)
+    );
+  });
+  it("loadProject fails when no project ever saved", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    return loadProject(p1.id, sourcecredDirectory).catch((e) =>
+      expect(e).toMatch(`project ${p1.id} not loaded`)
+    );
+  });
+  it("loadProject fails when a different project was saved", async () => {
+    const sourcecredDirectory = tmp.dirSync().name;
+    await setupProjectDirectory(p2, sourcecredDirectory);
+    return loadProject(p1.id, sourcecredDirectory).catch((e) =>
+      expect(e).toMatch(`project ${p1.id} not loaded`)
+    );
+  });
+});

yarn.lock

@@ -1678,6 +1678,11 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
   integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
 
+base-64@^0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb"
+  integrity sha1-eAqZyE59YAJgNhURxId2E78k9rs=
+
 base64-js@^1.0.2:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.0.tgz#cab1e6118f051095e58b5281aea8c1cd22bfc0e3"