From 9b105ee4ce81fff7cbed0a10f3b3c7fec4ec53d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dandelion=20Man=C3=A9?= Date: Thu, 18 Jul 2019 18:45:49 +0100 Subject: [PATCH] Add `core/project` and `core/project_io` This creates a new `Project` type which will replace `RepoId` as the index type for saving and loading data. The basic data type is added to `project.js`. Rather than having a `RepoIdRegistry`, I intend to infer the registry at build time by scanning for available projects saved in the sourcecred directory. I've added the `project_io` module for this task. It has methods for setting up a project subdirectory, and loading the `Project` info from that subdirectory. To ensure that projects ids can be encoded even if they have symbols like `/` and `@`, we base64 encode them. To ensure that project ids can be retrieved at build time, the `getProjectIds` method is factored out into its own plain ECMAScript module. For all non-build time needs, it is re-exported from `project_io`. Test plan: Unit tests added; run `yarn test`. --- package.json | 1 + src/core/_getProjectIds.js | 36 ++++++++++ src/core/project.js | 47 +++++++++++++ src/core/project.test.js | 29 ++++++++ src/core/project_io.js | 94 ++++++++++++++++++++++++++ src/core/project_io.test.js | 129 ++++++++++++++++++++++++++++++++++++ yarn.lock | 5 ++ 7 files changed, 341 insertions(+) create mode 100644 src/core/_getProjectIds.js create mode 100644 src/core/project.js create mode 100644 src/core/project.test.js create mode 100644 src/core/project_io.js create mode 100644 src/core/project_io.test.js diff --git a/package.json b/package.json index 09a6da8..3c4511b 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "private": true, "dependencies": { "aphrodite": "^2.1.0", + "base-64": "^0.1.0", "better-sqlite3": "^5.4.0", "chalk": "2.4.2", "commonmark": "^0.29.0", diff --git a/src/core/_getProjectIds.js b/src/core/_getProjectIds.js new file mode 100644 index 0000000..89eadc2 --- /dev/null +++ b/src/core/_getProjectIds.js @@ -0,0 +1,36 @@ +// @flow + +// This file is a complement to `./project_io.js`. It contains the +// implementation of `getProjectIds`, which is written in plain ECMAScript so +// that we can depend on it at build time. Regular users should not depend on +// this file; instead, depend on `./project_io.js`, which re-exports this +// method. +// +// This file is tested in ./project_io.test.js + +const path = require("path"); +const base64 = require("base-64"); +const fs = require("fs-extra"); + +module.exports = function getProjectIds( + sourcecredDirectory /*: string */ +) /*: $ReadOnlyArray */ { + const projectsPath = path.join(sourcecredDirectory, "projects"); + let entries = []; + try { + entries = fs.readdirSync(projectsPath); + } catch { + return []; + } + const projectIds = []; + for (const entry of entries) { + const jsonPath = path.join(projectsPath, entry, "project.json"); + try { + fs.statSync(jsonPath); + projectIds.push(base64.decode(entry)); + } catch { + continue; + } + } + return projectIds; +}; diff --git a/src/core/project.js b/src/core/project.js new file mode 100644 index 0000000..b22a525 --- /dev/null +++ b/src/core/project.js @@ -0,0 +1,47 @@ +// @flow + +import base64 from "base-64"; +import {type RepoId} from "../core/repoId"; +import {toCompat, fromCompat, type Compatible} from "../util/compat"; + +export type ProjectId = string; + +/** + * A project represents a scope for cred analysis. + * + * Right now it has an `id` (which should be unique across a user's projects) + * and an array of GitHub RepoIds. + * + * In the future, we will add support for more plugins (and remove the + * hardcoded GitHub support). + * + * We may add more fields (e.g. a description) to this object in the futre. + * + * We may create a complimentary object with load/cache info for the project in + * the future (e.g. showing the last update time for each of the project's data + * dependencies). + */ +export type Project = {| + +id: ProjectId, + +repoIds: $ReadOnlyArray, +|}; + +const COMPAT_INFO = {type: "sourcecred/project", version: "0.1.0"}; + +export type ProjectJSON = Compatible; + +export function projectToJSON(p: Project): ProjectJSON { + return toCompat(COMPAT_INFO, p); +} + +export function projectFromJSON(j: ProjectJSON): Project { + return fromCompat(COMPAT_INFO, j); +} + +/** + * Encode the project ID so it can be stored on the filesystem, + * or retrieved via XHR from the frontend. + */ +export function encodeProjectId(id: ProjectId): string { + return base64.encode(id); +} diff --git a/src/core/project.test.js b/src/core/project.test.js new file mode 100644 index 0000000..90c5f53 --- /dev/null +++ b/src/core/project.test.js @@ -0,0 +1,29 @@ +// @flow + +import {projectToJSON, projectFromJSON, type Project} from "./project"; + +import {makeRepoId} from "./repoId"; + +describe("core/project.js", () => { + const foobar = Object.freeze(makeRepoId("foo", "bar")); + const foozod = Object.freeze(makeRepoId("foo", "zod")); + const p1: Project = Object.freeze({ + id: "foo/bar", + repoIds: Object.freeze([foobar]), + }); + const p2: Project = Object.freeze({ + id: "@foo", + repoIds: Object.freeze([foobar, foozod]), + }); + describe("to/fro JSON", () => { + it("round trip is identity", () => { + function check(p: Project) { + const json = projectToJSON(p); + const p_ = projectFromJSON(json); + expect(p).toEqual(p_); + } + check(p1); + check(p2); + }); + }); +}); diff --git a/src/core/project_io.js b/src/core/project_io.js new file mode 100644 index 0000000..1bf121f --- /dev/null +++ b/src/core/project_io.js @@ -0,0 +1,94 @@ +// @flow +// This module contains logic for loading/saving projects to the +// sourcecred directory. +// +// It is separated from project.js so that it's possible to depend on project +// logic independent from anything related to the filesystem. (Depending +// on the path or fs module in the frontend would create a build error.) + +import fs from "fs-extra"; +import path from "path"; +import stringify from "json-stable-stringify"; + +import { + type Project, + type ProjectId, + projectToJSON, + projectFromJSON, + encodeProjectId, +} from "./project"; +import _getProjectIds from "./_getProjectIds"; + +/** + * Get the ids for every project saved on the filesystem. + * + * It is not guaranteed that it will be possible to load the id in question. + * (For example, the project may be malformed, or may have an outdated compat + * version.) + */ +export function getProjectIds( + sourcecredDirectory: string +): $ReadOnlyArray { + return _getProjectIds(sourcecredDirectory); +} + +/** + * Returns the project directory for the given id. + * + * Does not guarantee that the project directory has been created; + * does not do any IO. + */ +export function directoryForProjectId( + id: ProjectId, + sourcecredDirectory: string +): string { + return path.join(sourcecredDirectory, "projects", encodeProjectId(id)); +} + +/** + * Sets up a directory for the project, including a `project.json` file describing the project. + * + * If there is already a project.json file present, it will be over-written. + * + * Returns the project directory. + */ +export async function setupProjectDirectory( + project: Project, + sourcecredDirectory: string +): Promise { + const projectDirectory = directoryForProjectId( + project.id, + sourcecredDirectory + ); + await fs.mkdirp(projectDirectory); + const projectFile = path.join(projectDirectory, "project.json"); + await fs.writeFile(projectFile, stringify(projectToJSON(project))); + return projectDirectory; +} + +/** + * Load the Project with given id from the sourcecred directory. + * + * This method may throw an error if the project is not present, + * or is malformed or corrupted. + */ +export async function loadProject( + id: ProjectId, + sourcecredDirectory: string +): Promise { + const directory = directoryForProjectId(id, sourcecredDirectory); + const jsonPath = path.join(directory, "project.json"); + try { + const contents = await fs.readFile(jsonPath); + const project: Project = projectFromJSON(JSON.parse(contents)); + if (project.id !== id) { + throw new Error(`project ${project.id} saved under id ${id}`); + } + return project; + } catch (e) { + if (e.message.startsWith("ENOENT:")) { + throw `project ${id} not loaded`; + } + throw e; + } +} diff --git a/src/core/project_io.test.js b/src/core/project_io.test.js new file mode 100644 index 0000000..7ed254a --- /dev/null +++ b/src/core/project_io.test.js @@ -0,0 +1,129 @@ +// @flow + +import tmp from "tmp"; +import path from "path"; +import fs from "fs-extra"; + +import {type Project, encodeProjectId, projectToJSON} from "./project"; + +import { + getProjectIds, + setupProjectDirectory, + directoryForProjectId, + loadProject, +} from "./project_io"; + +import {makeRepoId} from "./repoId"; + +describe("core/project_io.js", () => { + const foobar = Object.freeze(makeRepoId("foo", "bar")); + const foozod = Object.freeze(makeRepoId("foo", "zod")); + const p1: Project = Object.freeze({ + id: "foo/bar", + repoIds: Object.freeze([foobar]), + }); + const p2: Project = Object.freeze({ + id: "@foo", + repoIds: Object.freeze([foobar, foozod]), + }); + + it("setupProjectDirectory results in a loadable project", async () => { + const sourcecredDirectory = tmp.dirSync().name; + await setupProjectDirectory(p1, sourcecredDirectory); + const ps = getProjectIds(sourcecredDirectory); + expect(ps).toEqual([p1.id]); + expect(await loadProject(p1.id, sourcecredDirectory)).toEqual(p1); + }); + it("setupProjectDirectory twice results in two loadable projects", async () => { + const sourcecredDirectory = tmp.dirSync().name; + await setupProjectDirectory(p1, sourcecredDirectory); + await setupProjectDirectory(p2, sourcecredDirectory); + const ps = getProjectIds(sourcecredDirectory); + expect(ps).toHaveLength(2); + expect(ps.slice().sort()).toEqual([p2.id, p1.id]); + expect(await loadProject(p1.id, sourcecredDirectory)).toEqual(p1); + expect(await loadProject(p2.id, sourcecredDirectory)).toEqual(p2); + }); + it("getProjectIds returns no projects if none were setup", async () => { + const sourcecredDirectory = tmp.dirSync().name; + const ps = getProjectIds(sourcecredDirectory); + expect(ps).toHaveLength(0); + }); + it("setupProjectDirectory returns the right directory", async () => { + const sourcecredDirectory = tmp.dirSync().name; + const dir = await setupProjectDirectory(p1, sourcecredDirectory); + expect(dir).toEqual(directoryForProjectId(p1.id, sourcecredDirectory)); + const projectJsonPath = path.join(dir, "project.json"); + await fs.stat(projectJsonPath); + }); + it("projects can be accessed using the encoded ID", async () => { + // Necessary so that frontend consumers can locate the project via the file mirror API + const sourcecredDirectory = tmp.dirSync().name; + await setupProjectDirectory(p1, sourcecredDirectory); + const projectJsonPath = path.join( + sourcecredDirectory, + "projects", + encodeProjectId(p1.id), + "project.json" + ); + await fs.stat(projectJsonPath); + }); + it("getProjectIds ignores non-project subdirectories", async () => { + const sourcecredDirectory = tmp.dirSync().name; + await setupProjectDirectory(p1, sourcecredDirectory); + await fs.mkdirp(path.join(sourcecredDirectory, "projects", "foobar")); + const ps = getProjectIds(sourcecredDirectory); + expect(ps).toEqual([p1.id]); + }); + it("getProjectIds ignores non-project file entries", async () => { + const sourcecredDirectory = tmp.dirSync().name; + await setupProjectDirectory(p1, sourcecredDirectory); + fs.writeFileSync( + path.join(sourcecredDirectory, "projects", "foobar"), + "1234" + ); + const ps = getProjectIds(sourcecredDirectory); + expect(ps).toEqual([p1.id]); + }); + it("loadProject throws an error on inconsistent id", async () => { + const sourcecredDirectory = tmp.dirSync().name; + const projectDirectory = await setupProjectDirectory( + p1, + sourcecredDirectory + ); + const jsonPath = path.join(projectDirectory, "project.json"); + const badJson = projectToJSON(p2); + fs.writeFileSync(jsonPath, JSON.stringify(badJson)); + expect.assertions(1); + return loadProject(p1.id, sourcecredDirectory).catch((e) => + expect(e.message).toMatch(`project ${p2.id} saved under id ${p1.id}`) + ); + }); + it("loadProject throws an error on bad compat", async () => { + const sourcecredDirectory = tmp.dirSync().name; + const projectDirectory = await setupProjectDirectory( + p1, + sourcecredDirectory + ); + const jsonPath = path.join(projectDirectory, "project.json"); + const badJson = [{type: "sourcecred/project", version: "NaN"}, {}]; + fs.writeFileSync(jsonPath, JSON.stringify(badJson)); + expect.assertions(1); + return loadProject(p1.id, sourcecredDirectory).catch((e) => + expect(e.message).toMatch(`tried to load unsupported version`) + ); + }); + it("loadProject fails when no project ever saved", async () => { + const sourcecredDirectory = tmp.dirSync().name; + return loadProject(p1.id, sourcecredDirectory).catch((e) => + expect(e).toMatch(`project ${p1.id} not loaded`) + ); + }); + it("loadProject fails when a different project was saved", async () => { + const sourcecredDirectory = tmp.dirSync().name; + await setupProjectDirectory(p2, sourcecredDirectory); + return loadProject(p1.id, sourcecredDirectory).catch((e) => + expect(e).toMatch(`project ${p1.id} not loaded`) + ); + }); +}); diff --git a/yarn.lock b/yarn.lock index 946d66f..f5541d1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1678,6 +1678,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= +base-64@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb" + integrity sha1-eAqZyE59YAJgNhURxId2E78k9rs= + base64-js@^1.0.2: version "1.3.0" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.0.tgz#cab1e6118f051095e58b5281aea8c1cd22bfc0e3"