Add the CredAccounts data format (#2017)

This commit adds a new intermediate data format which includes a mixture
of account data and cred data. It includes the full info for users where
we have identities and accounts, and also includes info on "unclaimed"
aliases (user accounts not linked to any identity).

The non-alias data is useful for computing Grain distributions which is
why I've prioritized it, but I'm also writing the data out to disk
because I think it might prove useful, either for frontend development
or for external consumers. It's definitely an experimental API so folks
shouldn't assume it will stay around unchanged indefinitely.

Test plan: Inspected the output, also added tests.

src/cli/score.js

@@ -20,12 +20,14 @@ import {
   toJSON as credResultToJSON,
   compressByThreshold,
 } from "../analysis/credResult";
+import {CredView} from "../analysis/credView";
 import * as Params from "../analysis/timeline/params";
 import {
   contractions as identityContractions,
   declaration as identityDeclaration,
 } from "../ledger/identity";
 import {Ledger, parser as ledgerParser} from "../ledger/ledger";
+import {computeCredAccounts} from "../ledger/credAccounts";
 
 function die(std, message) {
   std.err("fatal: " + message);
@@ -105,6 +107,15 @@ const scoreCommand: Command = async (args, std) => {
   const credJSON = stringify(credResultToJSON(compressed));
   const outputPath = pathJoin(baseDir, "output", "credResult.json");
   await fs.writeFile(outputPath, credJSON);
+
+  // Write out the account data for convenient usage.
+  // Note: this is an experimental format and may change or get
+  // removed in the future.
+  const credView = new CredView(credResult);
+  const credAccounts = computeCredAccounts(ledger, credView);
+  const accountsPath = pathJoin(baseDir, "output", "accounts.json");
+  await fs.writeFile(accountsPath, stringify(credAccounts));
+
   taskReporter.finish("score");
   return 0;
 };

src/ledger/credAccounts.js

@@ -0,0 +1,113 @@
+// @flow
+
+/**
+ * This module outputs aggregated data that combines Cred Scores with Ledger
+ * Account data.
+ *
+ * We use this internally when creating Grain distributions using a Ledger and
+ * a Cred View. It's also an experimental output format which gives overall
+ * information on the cred in an instance. We may remove it or make breaking
+ * changes to it in the future.
+ */
+import {sum} from "d3-array";
+import {Ledger, type Account} from "./ledger";
+import {CredView} from "../analysis/credView";
+import {type TimestampMs} from "../util/timestamp";
+import {NodeAddress, type NodeAddressT} from "../core/graph";
+
+export type Cred = $ReadOnlyArray<number>;
+
+export type CredAccount = {|
+  +cred: Cred,
+  +totalCred: number,
+  +account: Account,
+|};
+
+export type UnclaimedAlias = {|
+  +address: NodeAddressT,
+  // We include the description for convenience in figuring out who this user is,
+  // rendering in a UI, etc. This is just the description from the Graph.
+  +description: string,
+  +totalCred: number,
+  +cred: Cred,
+|};
+
+export type CredAccountData = {|
+  // Regular accounts: an identity with Cred, and potentially Grain
+  +accounts: $ReadOnlyArray<CredAccount>,
+  // Unclaimed aliases: An account on some platform that hasn't yet been
+  // connected to any SourceCred identity
+  +unclaimedAliases: $ReadOnlyArray<UnclaimedAlias>,
+  // The timestamps demarcating the ends of the Cred intervals.
+  // For interpreting the Cred data associated with cred accounts and
+  // unclaimed accounts.
+  +intervalEndpoints: $ReadOnlyArray<TimestampMs>,
+|};
+
+export function computeCredAccounts(
+  ledger: Ledger,
+  credView: CredView
+): CredAccountData {
+  const grainAccounts = ledger.accounts();
+  const userlikeInfo = new Map();
+  for (const {address, credOverTime, description} of credView.userNodes()) {
+    if (credOverTime == null) {
+      throw new Error(
+        `userlike ${NodeAddress.toString(address)} does not have detailed cred`
+      );
+    }
+    userlikeInfo.set(address, {cred: credOverTime.cred, description});
+  }
+  const intervalEndpoints = credView.credResult().credData.intervalEnds;
+  return _computeCredAccounts(grainAccounts, userlikeInfo, intervalEndpoints);
+}
+
+export function _computeCredAccounts(
+  grainAccounts: $ReadOnlyArray<Account>,
+  userlikeInfo: Map<NodeAddressT, {|+cred: Cred, +description: string|}>,
+  intervalEndpoints: $ReadOnlyArray<TimestampMs>
+): CredAccountData {
+  const aliases: Set<NodeAddressT> = new Set();
+  const accountAddresses: Set<NodeAddressT> = new Set();
+
+  const accounts = [];
+  const unclaimedAliases = [];
+
+  for (const account of grainAccounts) {
+    accountAddresses.add(account.identity.address);
+    for (const alias of account.identity.aliases) {
+      aliases.add(alias);
+    }
+    const info = userlikeInfo.get(account.identity.address);
+    if (info == null) {
+      throw new Error(
+        `cred sync error: no info for account ${account.identity.name}`
+      );
+    }
+    const {cred} = info;
+    const credAccount = {account, cred, totalCred: sum(cred)};
+    accounts.push(credAccount);
+  }
+
+  for (const [userAddress, info] of userlikeInfo.entries()) {
+    if (accountAddresses.has(userAddress)) {
+      // This userlike actually has an explicit account
+      continue;
+    }
+    if (aliases.has(userAddress)) {
+      throw new Error(
+        `cred sync error: alias ${NodeAddress.toString(
+          userAddress
+        )} included in Cred scores`
+      );
+    }
+    const {cred, description} = info;
+    unclaimedAliases.push({
+      address: userAddress,
+      cred,
+      totalCred: sum(cred),
+      description,
+    });
+  }
+  return {accounts, unclaimedAliases, intervalEndpoints};
+}

src/ledger/credAccounts.test.js

@@ -0,0 +1,79 @@
+// @flow
+
+import {NodeAddress} from "../core/graph";
+import {_computeCredAccounts} from "./credAccounts";
+import {Ledger} from "./ledger";
+
+describe("ledger/credAccounts", () => {
+  describe("_computeCredAccounts", () => {
+    it("works in a simple case", () => {
+      const ledger = new Ledger();
+      ledger.createIdentity("USER", "sourcecred");
+      const account = ledger.accounts()[0];
+      const accountCred = [0, 1, 2];
+      const userCred = [1, 0, 1];
+      const userAddress = NodeAddress.empty;
+      const info = new Map([
+        [
+          account.identity.address,
+          {cred: accountCred, description: "irrelevant"},
+        ],
+        [userAddress, {cred: userCred, description: "Little lost user"}],
+      ]);
+      const intervalEndpoints = [123, 125, 127];
+      const expectedCredAccount = {cred: accountCred, account, totalCred: 3};
+      const expectedUnclaimedAccount = {
+        cred: userCred,
+        totalCred: 2,
+        address: userAddress,
+        description: "Little lost user",
+      };
+      const expectedData = {
+        accounts: [expectedCredAccount],
+        unclaimedAliases: [expectedUnclaimedAccount],
+        intervalEndpoints,
+      };
+      expect(_computeCredAccounts([account], info, intervalEndpoints)).toEqual(
+        expectedData
+      );
+    });
+    it("errors if an alias address is in the cred scores", () => {
+      const ledger = new Ledger();
+      const id = ledger.createIdentity("USER", "sourcecred");
+      const userAddress = NodeAddress.empty;
+      ledger.addAlias(id, userAddress);
+
+      const account = ledger.accounts()[0];
+      const accountCred = [0, 1, 2];
+      const userCred = [1, 0, 1];
+      const info = new Map([
+        [
+          account.identity.address,
+          {cred: accountCred, description: "irrelevant"},
+        ],
+        [userAddress, {cred: userCred, description: "irrelevant"}],
+      ]);
+      const intervalEndpoints = [123, 125, 127];
+
+      const thunk = () =>
+        _computeCredAccounts([account], info, intervalEndpoints);
+      expect(thunk).toThrowError(
+        `cred sync error: alias ${NodeAddress.toString(
+          userAddress
+        )} included in Cred scores`
+      );
+    });
+    it("errors if an account doesn't have cred info", () => {
+      const ledger = new Ledger();
+      ledger.createIdentity("USER", "sourcecred");
+
+      const account = ledger.accounts()[0];
+      const scores = new Map();
+      const intervalEndpoints = [123, 125, 127];
+
+      const thunk = () =>
+        _computeCredAccounts([account], scores, intervalEndpoints);
+      expect(thunk).toThrowError(`cred sync error: no info for account`);
+    });
+  });
+});