Hackily add support for mixed GitHub/Discourse projects (#1378)

For phase one of the CredSperiment, I need a SourceCred instance which combines GitHub and Discourse servers. I'll also need to be able to give it very specific configuration to collapse certain user identities together.

Shortly after launching the CredSperiment, I plan to come back and totally re-write SourceCred's command line interface and site building system, in a way that will throw away most of the existing codebase.

As such, I found it expedient to add rather hacky and untested support for loading combined GitHub/Discourse instances, so I can land the promised features. This PR does so by:

- adding sourcecred gen-project for constructing project.json files
- adding sourcecred load --project for loading a project.json file
- ensuring that load provides the right plugins based on the project that's in scope
- updating build_static_site so that it can use the new --project flag

Test plan:
I have done some end-to-end testing, but the overall commit stack lacks automated testing. This is a deliberate tradeoff: I'm planning to re-write this section of the codebase, and the testing ergonomics are not great, so I'd rather accept some technical debt, especially since I plan to pay it off soon.

See the pull request on GitHub for the individual constituent commits.

scripts/build_static_site.sh

@@ -4,6 +4,7 @@ set -eu
 usage() {
     printf 'usage: build_static_site.sh --target TARGET\n'
     printf '                            [--project PROJECT [...]]\n'
+    printf '                            [--project-file PROJECT_FILE [...]]\n'
     printf '                            [--weights WEIGHTS_FILE]\n'
     printf '                            [--cname DOMAIN]\n'
     printf '                            [--no-backend]\n'
@@ -15,6 +16,8 @@ usage() {
     printf '\t%s\n' 'an empty directory into which to build the site'
     printf '%s\n' '--project PROJECT'
     printf '\t%s\n' 'a project spec; see help for cli/load.js for details'
+    printf '%s\n' '--project-file PROJECT_FILE'
+    printf '\t%s\n' 'the path to a file containing a project config'
     printf '%s\n' '--weights WEIGHTS_FILE'
     printf '\t%s\n' 'path to a json file which contains a weights configuration.'
     printf '\t%s\n' 'This will be used instead of the default weights and persisted.'
@@ -55,6 +58,7 @@ parse_args() {
     weights=
     repos=( )
     projects=( )
+    project_files=( )
     while [ $# -gt 0 ]; do
         case "$1" in
             --target)
@@ -78,6 +82,11 @@ parse_args() {
                 if [ $# -eq 0 ]; then die 'missing value for --project'; fi
                 projects+=( "$1" )
                 ;;
+            --project-file)
+                shift
+                if [ $# -eq 0 ]; then die 'missing value for --project-file'; fi
+                project_files+=( "$1" )
+                ;;
             --cname)
                 shift
                 if [ $# -eq 0 ]; then die 'missing value for --cname'; fi
@@ -151,6 +160,17 @@ build() {
         done
     fi
 
+    if [ "${#project_files[@]}" -ne 0 ]; then
+        local weightsStr=""
+        if [ -n "${weights}" ]; then
+            weightsStr="--weights ${weights}"
+        fi
+        for project_file in "${project_files[@]}"; do
+            NODE_PATH="./node_modules${NODE_PATH:+:${NODE_PATH}}" \
+                node "${SOURCECRED_BIN:-./bin}/sourcecred.js" load --project "${project_file}" $weightsStr
+        done
+    fi
+
     yarn -s build --output-path "${target}"
 
     # Copy the SourceCred data into the appropriate API route. Using

src/cli/defaultPlugins.js

@@ -1,9 +0,0 @@
-// @flow
-
-import deepFreeze from "deep-freeze";
-import {type PluginDeclaration} from "../analysis/pluginDeclaration";
-import {declaration as githubDeclaration} from "../plugins/github/declaration";
-
-export const DEFAULT_PLUGINS: $ReadOnlyArray<PluginDeclaration> = deepFreeze([
-  githubDeclaration,
-]);

src/cli/genProject.js

@@ -0,0 +1,187 @@
+// @flow
+// Implementation of `sourcecred gen-project`.
+// This method is intended as a placeholder for generating a project definition,
+// before we build a more intentional declarative json config approach, as discussed
+// here: https://github.com/sourcecred/sourcecred/issues/1232#issuecomment-519538494
+// This method is untested; please take care when modifying it!
+
+import dedent from "../util/dedent";
+import type {Command} from "./command";
+import * as Common from "./common";
+import stringify from "json-stable-stringify";
+import {type Project, projectToJSON} from "../core/project";
+import {type RepoId} from "../core/repoId";
+import {specToProject} from "../plugins/github/specToProject";
+import * as NullUtil from "../util/null";
+
+function usage(print: (string) => void): void {
+  print(
+    dedent`\
+    usage: sourcecred gen-project PROJECT_ID
+                                  [--github GITHUB_SPEC [...]]
+                                  [--discourse-url DISCOURSE_URL]
+                                  [--discourse-username DISCOURSE_USERNAME]
+           sourcecred gen-project --help
+
+    Generates a SourceCred project configuration based on the provided specs.
+
+    A PROJECT_ID must be provided, and will be the name of the project.
+
+    Zero or more github specs may be provided; each GitHub spec can be of the
+    form OWNER/NAME (as in 'torvalds/linux') for loading a single repository,
+    or @owner (as in '@torvalds') for loading all repositories owned by a given
+    account.
+
+    A discourse url and discourse username may be provided. If one is provided,
+    then both must be. The discourse url is a url to a valid Discourse server,
+    as in 'https://discourse.sourcecred.io', and the username must be a valid
+    user on that server, as in 'credbot'. The user in question should not have
+    any special or admin permissions, so that it won't encounter hidden
+    messages.
+
+    All of the GitHub specs, and the Discourse specification (if it exists)
+    will be combined into a single project. The serialized project
+    configuration will be printed to stdout.
+
+    Arguments:
+        PROJECT_ID
+            Locally unique identifier for the project.
+
+        --github GITHUB_SPEC
+            A specification (in form 'OWNER/NAME' or '@OWNER') of GitHub
+            repositories to load.
+
+        --discourse-url DISCOURSE_URL
+            The url of a Discourse server to load.
+
+        --discourse-username DISCOURSE_USERNAME
+            The username of a Discourse account to scan from. It's recommended
+            to make an account called "credbot".
+
+        --help
+            Show this help message and exit, as 'sourcecred help scores'.
+
+    Environment Variables:
+        SOURCECRED_GITHUB_TOKEN
+            API token for GitHub. This should be a 40-character hex
+            string. Required if using GitHub specs.
+
+            To generate a token, create a "Personal access token" at
+            <https://github.com/settings/tokens>. When loading data for
+            public repositories, no special permissions are required.
+            For private repositories, the 'repo' scope is required.
+    `.trimRight()
+  );
+}
+
+function die(std, message) {
+  std.err("fatal: " + message);
+  std.err("fatal: run 'sourcecred help gen-project' for help");
+  return 1;
+}
+
+export const genProject: Command = async (args, std) => {
+  let projectId: string | null = null;
+  let discourseUrl: string | null = null;
+  let discourseUsername: string | null = null;
+  const githubSpecs: string[] = [];
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case "--help": {
+        usage(std.out);
+        return 0;
+      }
+      case "--github": {
+        if (++i >= args.length)
+          return die(std, "'--github' given without value");
+        githubSpecs.push(args[i]);
+        break;
+      }
+      case "--discourse-url": {
+        if (discourseUrl != undefined)
+          return die(std, "'--discourse-url' given multiple times");
+        if (++i >= args.length)
+          return die(std, "'--discourse-url' given without value");
+        discourseUrl = args[i];
+        break;
+      }
+      case "--discourse-username": {
+        if (discourseUsername != undefined)
+          return die(std, "'--discourse-username' given multiple times");
+        if (++i >= args.length)
+          return die(std, "'--discourse-username' given without value");
+        discourseUsername = args[i];
+        break;
+      }
+      default: {
+        if (projectId != null) return die(std, "multiple project IDs provided");
+        projectId = args[i];
+        break;
+      }
+    }
+  }
+
+  if (projectId == null) {
+    return die(std, "no project ID provided");
+  }
+
+  const githubToken = Common.githubToken();
+  const project: Project = await createProject({
+    projectId,
+    githubSpecs,
+    discourseUsername,
+    discourseUrl,
+    githubToken,
+  });
+  const projectJSON = projectToJSON(project);
+  console.log(stringify(projectJSON));
+  return 0;
+};
+
+export async function createProject(opts: {|
+  +projectId: string,
+  +githubSpecs: $ReadOnlyArray<string>,
+  +discourseUsername: string | null,
+  +discourseUrl: string | null,
+  +githubToken: string | null,
+|}): Promise<Project> {
+  const {
+    projectId,
+    githubSpecs,
+    discourseUsername,
+    discourseUrl,
+    githubToken,
+  } = opts;
+  let repoIds: RepoId[] = [];
+  let discourseServer = null;
+  if (discourseUrl && discourseUsername) {
+    discourseServer = {serverUrl: discourseUrl, apiUsername: discourseUsername};
+  } else if (
+    (!discourseUrl && discourseUsername) ||
+    (discourseUrl && !discourseUsername)
+  ) {
+    throw new Error(
+      "If either of discourseUrl and discourseUsername are provided, then both must be."
+    );
+  }
+  if (githubSpecs.length && githubToken == null) {
+    throw new Error("Provided GitHub specs without GitHub token.");
+  }
+  for (const spec of githubSpecs) {
+    const subproject = await specToProject(spec, NullUtil.get(githubToken));
+    repoIds = repoIds.concat(subproject.repoIds);
+  }
+  return {id: projectId, repoIds, discourseServer};
+}
+
+export default genProject;
+
+export const help: Command = async (args, std) => {
+  if (args.length === 0) {
+    usage(std.out);
+    return 0;
+  } else {
+    usage(std.err);
+    return 1;
+  }
+};

src/cli/help.js

@@ -7,6 +7,7 @@ import dedent from "../util/dedent";
 import {help as loadHelp} from "./load";
 import {help as scoresHelp} from "./scores";
 import {help as clearHelp} from "./clear";
+import {help as genProjectHelp} from "./genProject";
 
 const help: Command = async (args, std) => {
   if (args.length === 0) {
@@ -19,6 +20,7 @@ const help: Command = async (args, std) => {
     load: loadHelp,
     scores: scoresHelp,
     clear: clearHelp,
+    "gen-project": genProjectHelp,
   };
   if (subHelps[command] !== undefined) {
     return subHelps[command](args.slice(1), std);
@@ -39,6 +41,8 @@ function usage(print: (string) => void): void {
     Commands:
       load          load repository data into SourceCred
       clear         clear SoucrceCred data
+      scores        print SourceCred scores to stdout
+      gen-project   print a SourceCred project config to stdout
       help          show this help message
 
     Use 'sourcecred help COMMAND' for help about an individual command.

src/cli/load.js

@@ -5,17 +5,22 @@ import dedent from "../util/dedent";
 import {LoggingTaskReporter} from "../util/taskReporter";
 import type {Command} from "./command";
 import * as Common from "./common";
-import {defaultWeights} from "../analysis/weights";
+import {defaultWeights, fromJSON as weightsFromJSON} from "../analysis/weights";
+import {projectFromJSON} from "../core/project";
 import {load} from "../api/load";
 import {specToProject} from "../plugins/github/specToProject";
+import fs from "fs-extra";
 import {partialParams} from "../analysis/timeline/params";
-import {DEFAULT_PLUGINS} from "./defaultPlugins";
+import {type PluginDeclaration} from "../analysis/pluginDeclaration";
+import {declaration as discourseDeclaration} from "../plugins/discourse/declaration";
+import {declaration as githubDeclaration} from "../plugins/github/declaration";
 
 function usage(print: (string) => void): void {
   print(
     dedent`\
     usage: sourcecred load [PROJECT_SPEC...]
                            [--weights WEIGHTS_FILE]
+                           [--project PROJECT_FILE]
            sourcecred load --help
 
     Load a target project, generating a cred attribution for it.
@@ -29,6 +34,10 @@ function usage(print: (string) => void): void {
         PROJECT_SPEC:
             Identifier of a project to load.
 
+        --project PROJECT_FILE
+            Path to a json file which contains a project configuration.
+            That project will be loaded.
+
         --weights WEIGHTS_FILE
             Path to a json file which contains a weights configuration.
             This will be used instead of the default weights and persisted.
@@ -65,6 +74,7 @@ function die(std, message) {
 
 const loadCommand: Command = async (args, std) => {
   const projectSpecs: string[] = [];
+  const projectPaths: string[] = [];
   let weightsPath: ?string;
   for (let i = 0; i < args.length; i++) {
     switch (args[i]) {
@@ -80,19 +90,25 @@ const loadCommand: Command = async (args, std) => {
         weightsPath = args[i];
         break;
       }
+      case "--project": {
+        if (++i >= args.length)
+          return die(std, "'--project' given without value");
+        projectPaths.push(args[i]);
+        break;
+      }
       default: {
         projectSpecs.push(args[i]);
         break;
       }
     }
   }
-  if (projectSpecs.length == 0) {
+  if (projectSpecs.length === 0 && projectPaths.length === 0) {
     return die(std, "projects not specified");
   }
 
   let weights = defaultWeights();
   if (weightsPath) {
-    weights = await Common.loadWeights(weightsPath);
+    weights = await loadWeightOverrides(weightsPath);
   }
 
   const githubToken = Common.githubToken();
@@ -102,19 +118,29 @@ const loadCommand: Command = async (args, std) => {
 
   const taskReporter = new LoggingTaskReporter();
 
-  const projects = await Promise.all(
+  const specProjects = await Promise.all(
     projectSpecs.map((s) => specToProject(s, githubToken))
   );
   const params = partialParams({weights});
-  const plugins = DEFAULT_PLUGINS;
+  const manualProjects = await Promise.all(projectPaths.map(loadProject));
-  const optionses = projects.map((project) => ({
+  const projects = specProjects.concat(manualProjects);
-    project,
+  const optionses = projects.map((project) => {
-    params,
+    const plugins: PluginDeclaration[] = [];
-    plugins,
+    if (project.discourseServer != null) {
-    sourcecredDirectory: Common.sourcecredDirectory(),
+      plugins.push(discourseDeclaration);
-    githubToken,
+    }
-    discourseKey: Common.discourseKey(),
+    if (project.repoIds.length) {
-  }));
+      plugins.push(githubDeclaration);
+    }
+    return {
+      project,
+      params,
+      plugins,
+      sourcecredDirectory: Common.sourcecredDirectory(),
+      githubToken,
+      discourseKey: Common.discourseKey(),
+    };
+  });
   // Deliberately load in serial because GitHub requests that their API not
   // be called concurrently
   for (const options of optionses) {
@@ -123,6 +149,34 @@ const loadCommand: Command = async (args, std) => {
   return 0;
 };
 
+const loadWeightOverrides = async (path: string) => {
+  if (!(await fs.exists(path))) {
+    throw new Error("Could not find the weights file");
+  }
+
+  const raw = await fs.readFile(path, "utf-8");
+  const weightsJSON = JSON.parse(raw);
+  try {
+    return weightsFromJSON(weightsJSON);
+  } catch (e) {
+    throw new Error(`provided weights file is invalid:\n${e}`);
+  }
+};
+
+const loadProject = async (path: string) => {
+  if (!(await fs.exists(path))) {
+    throw new Error(`Project path ${path} does not exist`);
+  }
+
+  const raw = await fs.readFile(path, "utf-8");
+  const json = JSON.parse(raw);
+  try {
+    return projectFromJSON(json);
+  } catch (e) {
+    throw new Error(`project at path ${path} is invalid:\n${e}`);
+  }
+};
+
 export const help: Command = async (args, std) => {
   if (args.length === 0) {
     usage(std.out);

src/cli/load.test.js

@@ -10,8 +10,8 @@ import loadCommand, {help} from "./load";
 import type {LoadOptions} from "../api/load";
 import {defaultWeights, toJSON as weightsToJSON} from "../analysis/weights";
 import * as Common from "./common";
-import {DEFAULT_PLUGINS} from "./defaultPlugins";
 import {defaultParams, partialParams} from "../analysis/timeline/params";
+import {declaration as githubDeclaration} from "../plugins/github/declaration";
 
 import {makeRepoId, stringToRepoId} from "../core/repoId";
 
@@ -78,8 +78,8 @@ describe("cli/load", () => {
           repoIds: [makeRepoId("foo", "bar")],
           discourseServer: null,
         },
-        plugins: DEFAULT_PLUGINS,
         params: defaultParams(),
+        plugins: [githubDeclaration],
         sourcecredDirectory: Common.sourcecredDirectory(),
         githubToken: fakeGithubToken,
         discourseKey: fakeDiscourseKey,
@@ -104,7 +104,7 @@ describe("cli/load", () => {
           discourseServer: null,
         },
         params: defaultParams(),
-        plugins: DEFAULT_PLUGINS,
+        plugins: [githubDeclaration],
         sourcecredDirectory: Common.sourcecredDirectory(),
         githubToken: fakeGithubToken,
         discourseKey: fakeDiscourseKey,
@@ -142,7 +142,7 @@ describe("cli/load", () => {
           discourseServer: null,
         },
         params: partialParams({weights}),
-        plugins: DEFAULT_PLUGINS,
+        plugins: [githubDeclaration],
         sourcecredDirectory: Common.sourcecredDirectory(),
         githubToken: fakeGithubToken,
         discourseKey: fakeDiscourseKey,

src/cli/sourcecred.js

@@ -9,6 +9,7 @@ import help from "./help";
 import load from "./load";
 import scores from "./scores";
 import clear from "./clear";
+import genProject from "./genProject";
 
 const sourcecred: Command = async (args, std) => {
   if (args.length === 0) {
@@ -28,6 +29,8 @@ const sourcecred: Command = async (args, std) => {
       return clear(args.slice(1), std);
     case "scores":
       return scores(args.slice(1), std);
+    case "gen-project":
+      return genProject(args.slice(1), std);
     default:
       std.err("fatal: unknown command: " + JSON.stringify(args[0]));
       std.err("fatal: run 'sourcecred help' for commands and usage");