Make PageRank functions asynchronous (#519)

Summary:
The PageRank functions can take a long time to compute. We’d like them
to not lock the browser, and we’d also like them to communicate with
their clients (e.g., to update a progress bar). This code updates
`findStationaryDistribution` and downstream `pagerank` to return
promises.

Test Plan:
Unit tests updated. The cred explorer (`yarn start`) still works.
Applying

```diff
diff --git a/src/core/attribution/markovChain.js b/src/core/attribution/markovChain.js
index 2acce9c..c7a7159 100644
--- a/src/core/attribution/markovChain.js
+++ b/src/core/attribution/markovChain.js
@@ -166,6 +166,7 @@ export function findStationaryDistribution(
           return;
         }
       } while (Date.now() - start < yieldAfterMs);
+      console.log("Yielding.");
       setTimeout(tick, 0);
     };
     tick();
```

causes the appropriate log messages to be printed in the browser—about
once every ten iterations for `sourcecred/sourcecred`.

wchargin-branch: asynchronous-pagerank
This commit is contained in:
William Chargin 2018-07-24 17:46:32 -07:00 committed by GitHub
parent 854fd817c0
commit 4435a3cfad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 144 additions and 102 deletions

View File

@ -93,14 +93,13 @@ export default class App extends React.Component<Props, State> {
<button
disabled={graphWithMetadata == null || edgeEvaluator == null}
onClick={() => {
setTimeout(() => {
if (graphWithMetadata == null || edgeEvaluator == null) {
throw new Error("Unexpected null value");
}
const {graph} = graphWithMetadata;
const pnd = pagerank(graph, edgeEvaluator, {
verbose: true,
});
if (graphWithMetadata == null || edgeEvaluator == null) {
throw new Error("Unexpected null value");
}
const {graph} = graphWithMetadata;
pagerank(graph, edgeEvaluator, {
verbose: true,
}).then((pnd) => {
const data = {graphWithMetadata, pnd};
// In case a new graph was loaded while waiting for
// PageRank.
@ -110,7 +109,7 @@ export default class App extends React.Component<Props, State> {
if (!stomped) {
this.setState({data});
}
}, 0);
});
}}
>
Run basic PageRank

View File

@ -27,7 +27,7 @@ require("../testUtil").configureEnzyme();
const COLUMNS = () => ["Description", "Contribution", "Score"];
function example() {
async function example() {
const graph = new Graph();
const nodes = {
fooAlpha: NodeAddress.fromParts(["foo", "a", "1"]),
@ -140,7 +140,7 @@ function example() {
},
];
const pnd = pagerank(graph, (_unused_Edge) => ({
const pnd = await pagerank(graph, (_unused_Edge) => ({
toWeight: 1,
froWeight: 1,
}));
@ -167,15 +167,15 @@ describe("app/credExplorer/PagerankTable", () => {
);
expect(enzymeToJSON(element)).toMatchSnapshot();
});
it("renders expected message with just adapters", () => {
const {adapters} = example();
it("renders expected message with just adapters", async () => {
const {adapters} = await example();
const element = shallow(
<PagerankTable pnd={null} adapters={adapters} maxEntriesPerList={1} />
);
expect(enzymeToJSON(element)).toMatchSnapshot();
});
it("throws an error if maxEntriesPerList not set", () => {
const {pnd, adapters} = example();
it("throws an error if maxEntriesPerList not set", async () => {
const {pnd, adapters} = await example();
expect(() =>
shallow(
<PagerankTable
@ -187,8 +187,8 @@ describe("app/credExplorer/PagerankTable", () => {
)
).toThrowError("maxEntriesPerList");
});
it("renders thead column order properly", () => {
const {pnd, adapters} = example();
it("renders thead column order properly", async () => {
const {pnd, adapters} = await example();
const element = shallow(
<PagerankTable pnd={pnd} adapters={adapters} maxEntriesPerList={1} />
);
@ -198,8 +198,8 @@ describe("app/credExplorer/PagerankTable", () => {
});
describe("has a filter select", () => {
function setup() {
const {pnd, adapters} = example();
async function setup() {
const {pnd, adapters} = await example();
const element = shallow(
<PagerankTable pnd={pnd} adapters={adapters} maxEntriesPerList={1} />
);
@ -207,16 +207,16 @@ describe("app/credExplorer/PagerankTable", () => {
const options = label.find("option");
return {pnd, adapters, element, label, options};
}
it("with expected label text", () => {
const {label} = setup();
it("with expected label text", async () => {
const {label} = await setup();
const filterText = label
.find("span")
.first()
.text();
expect(filterText).toMatchSnapshot();
});
it("with expected option groups", () => {
const {options} = setup();
it("with expected option groups", async () => {
const {options} = await setup();
const optionsJSON = options.map((o) => ({
valueString: NodeAddress.toString(o.prop("value")),
style: o.prop("style"),
@ -224,8 +224,8 @@ describe("app/credExplorer/PagerankTable", () => {
}));
expect(optionsJSON).toMatchSnapshot();
});
it("with the ability to filter nodes passed to NodeRowList", () => {
const {element, options} = setup();
it("with the ability to filter nodes passed to NodeRowList", async () => {
const {element, options} = await setup();
const option1 = options.at(1);
const value = option1.prop("value");
expect(value).not.toEqual(NodeAddress.empty);
@ -243,8 +243,8 @@ describe("app/credExplorer/PagerankTable", () => {
});
describe("creates a NodeRowList", () => {
function setup() {
const {adapters, pnd} = example();
async function setup() {
const {adapters, pnd} = await example();
const maxEntriesPerList = 1;
const element = shallow(
<PagerankTable
@ -256,13 +256,13 @@ describe("app/credExplorer/PagerankTable", () => {
const nrl = element.find("NodeRowList");
return {adapters, pnd, element, nrl, maxEntriesPerList};
}
it("with the correct SharedProps", () => {
const {nrl, adapters, pnd, maxEntriesPerList} = setup();
it("with the correct SharedProps", async () => {
const {nrl, adapters, pnd, maxEntriesPerList} = await setup();
const expectedSharedProps = {adapters, pnd, maxEntriesPerList};
expect(nrl.prop("sharedProps")).toEqual(expectedSharedProps);
});
it("including all nodes by default", () => {
const {nrl, pnd} = setup();
it("including all nodes by default", async () => {
const {nrl, pnd} = await setup();
const expectedNodes = Array.from(pnd.keys());
expect(nrl.prop("nodes")).toEqual(expectedNodes);
});
@ -273,8 +273,8 @@ describe("app/credExplorer/PagerankTable", () => {
function sortedByScore(nodes: $ReadOnlyArray<NodeAddressT>, pnd) {
return sortBy(nodes, (node) => -NullUtil.get(pnd.get(node)).score);
}
function setup(maxEntriesPerList: number = 100000) {
const {adapters, pnd} = example();
async function setup(maxEntriesPerList: number = 100000) {
const {adapters, pnd} = await example();
const nodes = sortedByScore(Array.from(pnd.keys()), pnd)
.reverse() // ascending order!
.filter((x) =>
@ -288,8 +288,8 @@ describe("app/credExplorer/PagerankTable", () => {
const element = shallow(component);
return {element, adapters, sharedProps, nodes};
}
it("creates `NodeRow`s with the right props", () => {
const {element, nodes, sharedProps} = setup();
it("creates `NodeRow`s with the right props", async () => {
const {element, nodes, sharedProps} = await setup();
const rows = element.find("NodeRow");
expect(rows).toHaveLength(nodes.length);
const rowNodes = rows.map((row) => row.prop("node"));
@ -300,9 +300,9 @@ describe("app/credExplorer/PagerankTable", () => {
expect(row.prop("sharedProps")).toEqual(sharedProps);
});
});
it("creates up to `maxEntriesPerList` `NodeRow`s", () => {
it("creates up to `maxEntriesPerList` `NodeRow`s", async () => {
const maxEntriesPerList = 1;
const {element, nodes, sharedProps} = setup(maxEntriesPerList);
const {element, nodes, sharedProps} = await setup(maxEntriesPerList);
expect(nodes.length).toBeGreaterThan(maxEntriesPerList);
const rows = element.find("NodeRow");
expect(rows).toHaveLength(maxEntriesPerList);
@ -312,12 +312,12 @@ describe("app/credExplorer/PagerankTable", () => {
sortedByScore(nodes, sharedProps.pnd).slice(0, maxEntriesPerList)
);
});
it("sorts its children by score", () => {
it("sorts its children by score", async () => {
const {
element,
nodes,
sharedProps: {pnd},
} = setup();
} = await setup();
expect(nodes).not.toEqual(sortedByScore(nodes, pnd));
const rows = element.find("NodeRow");
const rowNodes = rows.map((row) => row.prop("node"));
@ -326,19 +326,19 @@ describe("app/credExplorer/PagerankTable", () => {
});
describe("NodeRow", () => {
function setup() {
const {pnd, adapters, nodes} = example();
async function setup() {
const {pnd, adapters, nodes} = await example();
const sharedProps = {adapters, pnd, maxEntriesPerList: 123};
const node = nodes.bar1;
const component = <NodeRow node={node} sharedProps={sharedProps} />;
const element = shallow(component);
return {element, node, sharedProps};
}
it("renders the right number of columns", () => {
expect(setup().element.find("td")).toHaveLength(COLUMNS().length);
it("renders the right number of columns", async () => {
expect((await setup()).element.find("td")).toHaveLength(COLUMNS().length);
});
it("renders the node description", () => {
const {element} = setup();
it("renders the node description", async () => {
const {element} = await setup();
const expectedDescription = 'bar: NodeAddress["bar","a","1"]';
const descriptionColumn = COLUMNS().indexOf("Description");
expect(descriptionColumn).not.toEqual(-1);
@ -350,8 +350,8 @@ describe("app/credExplorer/PagerankTable", () => {
.text()
).toEqual(expectedDescription);
});
it("renders an empty contribution column", () => {
const {element} = setup();
it("renders an empty contribution column", async () => {
const {element} = await setup();
const contributionColumn = COLUMNS().indexOf("Contribution");
expect(contributionColumn).not.toEqual(-1);
expect(
@ -361,8 +361,8 @@ describe("app/credExplorer/PagerankTable", () => {
.text()
).toEqual("—");
});
it("renders a score column with the node's log-score", () => {
const {element, sharedProps, node} = setup();
it("renders a score column with the node's log-score", async () => {
const {element, sharedProps, node} = await setup();
const {score: rawScore} = NullUtil.get(sharedProps.pnd.get(node));
const expectedScore = (Math.log(rawScore) + 10).toFixed(2);
const contributionColumn = COLUMNS().indexOf("Score");
@ -374,12 +374,12 @@ describe("app/credExplorer/PagerankTable", () => {
.text()
).toEqual(expectedScore);
});
it("does not render children by default", () => {
const {element} = setup();
it("does not render children by default", async () => {
const {element} = await setup();
expect(element.find("ContributionRowList")).toHaveLength(0);
});
it('has a working "expand" button', () => {
const {element, sharedProps, node} = setup();
it('has a working "expand" button', async () => {
const {element, sharedProps, node} = await setup();
expect(element.find("button").text()).toEqual("+");
element.find("button").simulate("click");
@ -397,8 +397,8 @@ describe("app/credExplorer/PagerankTable", () => {
});
describe("ContributionRowList", () => {
function setup(maxEntriesPerList: number = 100000) {
const {adapters, pnd, nodes} = example();
async function setup(maxEntriesPerList: number = 100000) {
const {adapters, pnd, nodes} = await example();
const depth = 2;
const node = nodes.bar1;
const sharedProps = {adapters, pnd, maxEntriesPerList};
@ -412,8 +412,8 @@ describe("app/credExplorer/PagerankTable", () => {
const element = shallow(component);
return {element, depth, node, sharedProps};
}
it("creates `ContributionRow`s with the right props", () => {
const {element, depth, node, sharedProps} = setup();
it("creates `ContributionRow`s with the right props", async () => {
const {element, depth, node, sharedProps} = await setup();
const contributions = NullUtil.get(sharedProps.pnd.get(node))
.scoredContributions;
const rows = element.find("ContributionRow");
@ -429,9 +429,9 @@ describe("app/credExplorer/PagerankTable", () => {
}))
);
});
it("limits the number of rows by `maxEntriesPerList`", () => {
it("limits the number of rows by `maxEntriesPerList`", async () => {
const maxEntriesPerList = 1;
const {element, node, sharedProps} = setup(maxEntriesPerList);
const {element, node, sharedProps} = await setup(maxEntriesPerList);
const contributions = NullUtil.get(sharedProps.pnd.get(node))
.scoredContributions;
expect(contributions.length).toBeGreaterThan(maxEntriesPerList);
@ -448,8 +448,8 @@ describe("app/credExplorer/PagerankTable", () => {
});
describe("ContributionRow", () => {
function setup() {
const {pnd, adapters, nodes} = example();
async function setup() {
const {pnd, adapters, nodes} = await example();
const sharedProps = {adapters, pnd, maxEntriesPerList: 123};
const target = nodes.bar1;
const {scoredContributions} = NullUtil.get(pnd.get(target));
@ -471,18 +471,18 @@ describe("app/credExplorer/PagerankTable", () => {
const element = shallow(component);
return {element, depth, target, source, contribution, sharedProps};
}
it("renders the right number of columns", () => {
expect(setup().element.find("td")).toHaveLength(COLUMNS().length);
it("renders the right number of columns", async () => {
expect((await setup()).element.find("td")).toHaveLength(COLUMNS().length);
});
it("has proper depth-based styling", () => {
const {element} = setup();
it("has proper depth-based styling", async () => {
const {element} = await setup();
expect({
buttonStyle: element.find("button").prop("style"),
trStyle: element.find("tr").prop("style"),
}).toMatchSnapshot();
});
it("renders the source view", () => {
const {element, sharedProps, contribution} = setup();
it("renders the source view", async () => {
const {element, sharedProps, contribution} = await setup();
const descriptionColumn = COLUMNS().indexOf("Description");
expect(descriptionColumn).not.toEqual(-1);
const view = element
@ -495,8 +495,8 @@ describe("app/credExplorer/PagerankTable", () => {
contribution: contribution.contribution,
});
});
it("renders the contribution percentage", () => {
const {element, contribution, sharedProps, target} = setup();
it("renders the contribution percentage", async () => {
const {element, contribution, sharedProps, target} = await setup();
const contributionColumn = COLUMNS().indexOf("Contribution");
expect(contributionColumn).not.toEqual(-1);
const proportion =
@ -512,8 +512,8 @@ describe("app/credExplorer/PagerankTable", () => {
.text()
).toEqual(expectedText);
});
it("renders a score column with the source's log-score", () => {
const {element, contribution} = setup();
it("renders a score column with the source's log-score", async () => {
const {element, contribution} = await setup();
const expectedScore = (Math.log(contribution.sourceScore) + 10).toFixed(
2
);
@ -526,12 +526,12 @@ describe("app/credExplorer/PagerankTable", () => {
.text()
).toEqual(expectedScore);
});
it("does not render children by default", () => {
const {element} = setup();
it("does not render children by default", async () => {
const {element} = await setup();
expect(element.find("ContributionRowList")).toHaveLength(0);
});
it('has a working "expand" button', () => {
const {element, depth, sharedProps, source} = setup();
it('has a working "expand" button', async () => {
const {element, depth, sharedProps, source} = await setup();
expect(element.find("button").text()).toEqual("+");
element.find("button").simulate("click");
@ -550,8 +550,8 @@ describe("app/credExplorer/PagerankTable", () => {
});
describe("ContributionView", () => {
function setup() {
const {pnd, adapters, nodes} = example();
async function setup() {
const {pnd, adapters, nodes} = await example();
const {scoredContributions} = NullUtil.get(pnd.get(nodes.bar1));
const contributions = scoredContributions.map((sc) => sc.contribution);
function contributionByType(t) {
@ -578,13 +578,13 @@ describe("app/credExplorer/PagerankTable", () => {
syntheticContribution,
};
}
it("always renders exactly one `Badge`", () => {
it("always renders exactly one `Badge`", async () => {
const {
cvForContribution,
inContribution,
outContribution,
syntheticContribution,
} = setup();
} = await setup();
for (const contribution of [
syntheticContribution,
inContribution,
@ -593,8 +593,8 @@ describe("app/credExplorer/PagerankTable", () => {
expect(cvForContribution(contribution).find("Badge")).toHaveLength(1);
}
});
it("for inward contributions, renders a `Badge` and description", () => {
const {cvForContribution, inContribution} = setup();
it("for inward contributions, renders a `Badge` and description", async () => {
const {cvForContribution, inContribution} = await setup();
const view = cvForContribution(inContribution);
const outerSpan = view.find("span").first();
const badge = outerSpan.find("Badge");
@ -602,8 +602,8 @@ describe("app/credExplorer/PagerankTable", () => {
expect(badge.children().text()).toEqual("is barred by");
expect(description.text()).toEqual('bar: NodeAddress["bar","a","1"]');
});
it("for outward contributions, renders a `Badge` and description", () => {
const {cvForContribution, outContribution} = setup();
it("for outward contributions, renders a `Badge` and description", async () => {
const {cvForContribution, outContribution} = await setup();
const view = cvForContribution(outContribution);
const outerSpan = view.find("span").first();
const badge = outerSpan.find("Badge");
@ -611,8 +611,8 @@ describe("app/credExplorer/PagerankTable", () => {
expect(badge.children().text()).toEqual("bars");
expect(description.text()).toEqual("xox node!");
});
it("for synthetic contributions, renders only a `Badge`", () => {
const {cvForContribution, syntheticContribution} = setup();
it("for synthetic contributions, renders only a `Badge`", async () => {
const {cvForContribution, syntheticContribution} = await setup();
const view = cvForContribution(syntheticContribution);
expect(view.find("span")).toHaveLength(0);
expect(

View File

@ -90,14 +90,14 @@ export function sparseMarkovChainAction(
return result;
}
export function findStationaryDistribution(
function* findStationaryDistributionGenerator(
chain: SparseMarkovChain,
options: {|
+verbose: boolean,
+convergenceThreshold: number,
+maxIterations: number,
|}
): Distribution {
): Generator<void, Distribution, void> {
let r0 = uniformDistribution(chain.length);
function computeDelta(pi0, pi1) {
let maxDelta = -Infinity;
@ -130,8 +130,44 @@ export function findStationaryDistribution(
}
return r0;
}
yield;
}
// ESLint knows that this next line is unreachable, but Flow doesn't. :-)
// eslint-disable-next-line no-unreachable
throw new Error("Unreachable.");
}
export function findStationaryDistribution(
chain: SparseMarkovChain,
options: {|
+verbose: boolean,
+convergenceThreshold: number,
+maxIterations: number,
+yieldAfterMs: number,
|}
): Promise<Distribution> {
let gen = findStationaryDistributionGenerator(chain, {
verbose: options.verbose,
convergenceThreshold: options.convergenceThreshold,
maxIterations: options.maxIterations,
});
return new Promise((resolve, _unused_reject) => {
const {yieldAfterMs} = options;
const tick = () => {
const start = Date.now();
do {
const result = gen.next();
if (result.done) {
if (result.value == null) {
// Should never happen.
throw new Error(String(result.value));
}
resolve(result.value);
return;
}
} while (Date.now() - start < yieldAfterMs);
setTimeout(tick, 0);
};
tick();
});
}

View File

@ -145,23 +145,24 @@ describe("core/attribution/markovChain", () => {
}
describe("findStationaryDistribution", () => {
it("finds an all-accumulating stationary distribution", () => {
it("finds an all-accumulating stationary distribution", async () => {
const chain = sparseMarkovChainFromTransitionMatrix([
[1, 0, 0],
[0.25, 0, 0.75],
[0.25, 0.75, 0],
]);
const pi = findStationaryDistribution(chain, {
const pi = await findStationaryDistribution(chain, {
maxIterations: 255,
convergenceThreshold: 1e-7,
verbose: false,
yieldAfterMs: 1,
});
expectStationary(chain, pi);
const expected = new Float64Array([1, 0, 0]);
expectAllClose(pi, expected);
});
it("finds a non-degenerate stationary distribution", () => {
it("finds a non-degenerate stationary distribution", async () => {
// Node 0 is the "center"; nodes 1 through 4 are "satellites". A
// satellite transitions to the center with probability 0.5, or to a
// cyclically adjacent satellite with probability 0.25 each. The
@ -173,34 +174,37 @@ describe("core/attribution/markovChain", () => {
[0.5, 0, 0.25, 0, 0.25],
[0.5, 0.25, 0, 0.25, 0],
]);
const pi = findStationaryDistribution(chain, {
const pi = await findStationaryDistribution(chain, {
maxIterations: 255,
convergenceThreshold: 1e-7,
verbose: false,
yieldAfterMs: 1,
});
expectStationary(chain, pi);
const expected = new Float64Array([1 / 3, 1 / 6, 1 / 6, 1 / 6, 1 / 6]);
expectAllClose(pi, expected);
});
it("finds the stationary distribution of a periodic chain", () => {
it("finds the stationary distribution of a periodic chain", async () => {
const chain = sparseMarkovChainFromTransitionMatrix([[0, 1], [1, 0]]);
const pi = findStationaryDistribution(chain, {
const pi = await findStationaryDistribution(chain, {
maxIterations: 255,
convergenceThreshold: 1e-7,
verbose: false,
yieldAfterMs: 1,
});
expectStationary(chain, pi);
const expected = new Float64Array([0.5, 0.5]);
expectAllClose(pi, expected);
});
it("returns initial distribution if maxIterations===0", () => {
it("returns initial distribution if maxIterations===0", async () => {
const chain = sparseMarkovChainFromTransitionMatrix([[0, 1], [0, 1]]);
const pi = findStationaryDistribution(chain, {
const pi = await findStationaryDistribution(chain, {
verbose: false,
convergenceThreshold: 1e-7,
maxIterations: 0,
yieldAfterMs: 1,
});
const expected = new Float64Array([0.5, 0.5]);
expect(pi).toEqual(expected);

View File

@ -35,11 +35,11 @@ function defaultOptions(): PagerankOptions {
};
}
export function pagerank(
export async function pagerank(
graph: Graph,
edgeWeight: EdgeEvaluator,
options?: PagerankOptions
): PagerankNodeDecomposition {
): Promise<PagerankNodeDecomposition> {
const fullOptions = {
...defaultOptions(),
...(options || {}),
@ -50,10 +50,11 @@ export function pagerank(
fullOptions.selfLoopWeight
);
const osmc = createOrderedSparseMarkovChain(contributions);
const distribution = findStationaryDistribution(osmc.chain, {
const distribution = await findStationaryDistribution(osmc.chain, {
verbose: fullOptions.verbose,
convergenceThreshold: fullOptions.convergenceThreshold,
maxIterations: fullOptions.maxIterations,
yieldAfterMs: 30,
});
const pi = distributionToNodeDistribution(osmc.nodeOrder, distribution);
return decompose(pi, contributions);

View File

@ -110,7 +110,7 @@ function validateDecomposition(decomposition) {
describe("core/attribution/contributions", () => {
describe("decompose", () => {
it("has the expected output on a simple asymmetric chain", () => {
it("has the expected output on a simple asymmetric chain", async () => {
const n1 = NodeAddress.fromParts(["n1"]);
const n2 = NodeAddress.fromParts(["n2"]);
const n3 = NodeAddress.fromParts(["sink"]);
@ -129,10 +129,11 @@ describe("core/attribution/contributions", () => {
const edgeWeight = () => ({toWeight: 6.0, froWeight: 3.0});
const contributions = createContributions(g, edgeWeight, 1.0);
const osmc = createOrderedSparseMarkovChain(contributions);
const pi = findStationaryDistribution(osmc.chain, {
const pi = await findStationaryDistribution(osmc.chain, {
verbose: false,
convergenceThreshold: 1e-6,
maxIterations: 255,
yieldAfterMs: 1,
});
const pr = distributionToNodeDistribution(osmc.nodeOrder, pi);
const result = decompose(pr, contributions);
@ -140,15 +141,16 @@ describe("core/attribution/contributions", () => {
validateDecomposition(result);
});
it("is valid on the example graph", () => {
it("is valid on the example graph", async () => {
const g = advancedGraph().graph1();
const edgeWeight = () => ({toWeight: 6.0, froWeight: 3.0});
const contributions = createContributions(g, edgeWeight, 1.0);
const osmc = createOrderedSparseMarkovChain(contributions);
const pi = findStationaryDistribution(osmc.chain, {
const pi = await findStationaryDistribution(osmc.chain, {
verbose: false,
convergenceThreshold: 1e-6,
maxIterations: 255,
yieldAfterMs: 1,
});
const pr = distributionToNodeDistribution(osmc.nodeOrder, pi);
const result = decompose(pr, contributions);