Use url encoding and make _getProjectIds async

Test plan: `yarn test --full` still passes. Also, I've ensured that the
async `_getProjectIds` is still usable in our webpack configs (via
modifying and testing the dependent commits).

package.json

@@ -4,7 +4,7 @@
   "private": true,
   "dependencies": {
     "aphrodite": "^2.1.0",
-    "base-64": "^0.1.0",
+    "base64url": "^3.0.1",
     "better-sqlite3": "^5.4.0",
     "chalk": "2.4.2",
     "commonmark": "^0.29.0",

src/core/_getProjectIds.js

@@ -9,28 +9,36 @@
 // This file is tested in ./project_io.test.js
 
 const path = require("path");
-const base64 = require("base-64");
+const base64url = require("base64url");
 const fs = require("fs-extra");
 
-module.exports = function getProjectIds(
+/**
+ * Get the ids for every project saved on the filesystem.
+ *
+ * It is not guaranteed that it will be possible to load the id in question.
+ * (For example, the project may be malformed, or may have an outdated compat
+ * version.)
+ */
+module.exports = async function getProjectIds(
   sourcecredDirectory /*: string */
-) /*: $ReadOnlyArray<string> */ {
+) /*: Promise<$ReadOnlyArray<string>> */ {
   const projectsPath = path.join(sourcecredDirectory, "projects");
   let entries = [];
   try {
-    entries = fs.readdirSync(projectsPath);
+    entries = await fs.readdir(projectsPath);
   } catch {
     return [];
   }
-  const projectIds = [];
-  for (const entry of entries) {
-    const jsonPath = path.join(projectsPath, entry, "project.json");
+  const getProjectId = async (entry) => {
     try {
-      fs.statSync(jsonPath);
-      projectIds.push(base64.decode(entry));
+      const jsonPath = path.join(projectsPath, entry, "project.json");
+      await fs.stat(jsonPath);
+      return base64url.decode(entry);
     } catch {
-      continue;
+      return null;
     }
-  }
-  return projectIds;
+  };
+
+  const maybeProjectIds = await Promise.all(entries.map(getProjectId));
+  return maybeProjectIds.filter((x) => x != null);
 };

src/core/project.js

@@ -1,6 +1,6 @@
 // @flow
 
-import base64 from "base-64";
+import base64url from "base64url";
 import {type RepoId} from "../core/repoId";
 import {toCompat, fromCompat, type Compatible} from "../util/compat";
 
@@ -43,5 +43,5 @@ export function projectFromJSON(j: ProjectJSON): Project {
  * or retrieved via XHR from the frontend.
  */
 export function encodeProjectId(id: ProjectId): string {
-  return base64.encode(id);
+  return base64url.encode(id);
 }

src/core/project.test.js

@@ -1,10 +1,16 @@
 // @flow
 
-import {projectToJSON, projectFromJSON, type Project} from "./project";
+import base64url from "base64url";
+import {
+  projectToJSON,
+  projectFromJSON,
+  type Project,
+  encodeProjectId,
+} from "./project";
 
 import {makeRepoId} from "./repoId";
 
-describe("core/project.js", () => {
+describe("core/project", () => {
   const foobar = Object.freeze(makeRepoId("foo", "bar"));
   const foozod = Object.freeze(makeRepoId("foo", "zod"));
   const p1: Project = Object.freeze({
@@ -26,4 +32,14 @@ describe("core/project.js", () => {
       check(p2);
     });
   });
+  describe("encodeProjectId", () => {
+    it("is a base64-url encoded id", () => {
+      const project = {id: "foo bar", repoIds: []};
+      const encoded = encodeProjectId(project.id);
+      expect(encoded).toEqual(base64url.encode("foo bar"));
+    });
+    it("is decodable to identity", () => {
+      expect(base64url.decode(encodeProjectId("foo bar"))).toEqual("foo bar");
+    });
+  });
 });

src/core/project_io.js

@@ -28,7 +28,7 @@ import _getProjectIds from "./_getProjectIds";
  */
 export function getProjectIds(
   sourcecredDirectory: string
-): $ReadOnlyArray<ProjectId> {
+): Promise<$ReadOnlyArray<ProjectId>> {
   return _getProjectIds(sourcecredDirectory);
 }
 

src/core/project_io.test.js

@@ -15,7 +15,7 @@ import {
 
 import {makeRepoId} from "./repoId";
 
-describe("core/project_io.js", () => {
+describe("core/project_io", () => {
   const foobar = Object.freeze(makeRepoId("foo", "bar"));
   const foozod = Object.freeze(makeRepoId("foo", "zod"));
   const p1: Project = Object.freeze({
@@ -30,7 +30,7 @@ describe("core/project_io.js", () => {
   it("setupProjectDirectory results in a loadable project", async () => {
     const sourcecredDirectory = tmp.dirSync().name;
     await setupProjectDirectory(p1, sourcecredDirectory);
-    const ps = getProjectIds(sourcecredDirectory);
+    const ps = await getProjectIds(sourcecredDirectory);
     expect(ps).toEqual([p1.id]);
     expect(await loadProject(p1.id, sourcecredDirectory)).toEqual(p1);
   });
@@ -38,7 +38,7 @@ describe("core/project_io.js", () => {
     const sourcecredDirectory = tmp.dirSync().name;
     await setupProjectDirectory(p1, sourcecredDirectory);
     await setupProjectDirectory(p2, sourcecredDirectory);
-    const ps = getProjectIds(sourcecredDirectory);
+    const ps = await getProjectIds(sourcecredDirectory);
     expect(ps).toHaveLength(2);
     expect(ps.slice().sort()).toEqual([p2.id, p1.id]);
     expect(await loadProject(p1.id, sourcecredDirectory)).toEqual(p1);
@@ -46,7 +46,7 @@ describe("core/project_io.js", () => {
   });
   it("getProjectIds returns no projects if none were setup", async () => {
     const sourcecredDirectory = tmp.dirSync().name;
-    const ps = getProjectIds(sourcecredDirectory);
+    const ps = await getProjectIds(sourcecredDirectory);
     expect(ps).toHaveLength(0);
   });
   it("setupProjectDirectory returns the right directory", async () => {
@@ -72,7 +72,7 @@ describe("core/project_io.js", () => {
     const sourcecredDirectory = tmp.dirSync().name;
     await setupProjectDirectory(p1, sourcecredDirectory);
     await fs.mkdirp(path.join(sourcecredDirectory, "projects", "foobar"));
-    const ps = getProjectIds(sourcecredDirectory);
+    const ps = await getProjectIds(sourcecredDirectory);
     expect(ps).toEqual([p1.id]);
   });
   it("getProjectIds ignores non-project file entries", async () => {
@@ -82,7 +82,7 @@ describe("core/project_io.js", () => {
       path.join(sourcecredDirectory, "projects", "foobar"),
       "1234"
     );
-    const ps = getProjectIds(sourcecredDirectory);
+    const ps = await getProjectIds(sourcecredDirectory);
     expect(ps).toEqual([p1.id]);
   });
   it("loadProject throws an error on inconsistent id", async () => {

yarn.lock

@@ -1678,16 +1678,16 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
   integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
 
-base-64@^0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb"
-  integrity sha1-eAqZyE59YAJgNhURxId2E78k9rs=
-
 base64-js@^1.0.2:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.0.tgz#cab1e6118f051095e58b5281aea8c1cd22bfc0e3"
   integrity sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==
 
+base64url@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d"
+  integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A==
+
 base@^0.11.1:
   version "0.11.2"
   resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f"