diff --git a/src/graphql/mirror.js b/src/graphql/mirror.js index a4a2ab6..e813b03 100644 --- a/src/graphql/mirror.js +++ b/src/graphql/mirror.js @@ -353,6 +353,58 @@ export class Mirror { addConnection.run({id, fieldname}); } } + + /** + * Find objects and connections that are not known to be up-to-date. + * + * An object is up-to-date if its own data has been loaded at least as + * recently as the provided date. + * + * A connection is up-to-date if it has been fetched at least as + * recently as the provided date, and at the time of fetching there + * were no more pages. + */ + _findOutdated(since: Date): QueryPlan { + const db = this._db; + return _inTransaction(db, () => { + const objects: $PropertyType = db + .prepare( + dedent`\ + SELECT typename AS typename, id AS id + FROM objects + LEFT OUTER JOIN updates ON objects.last_update = updates.rowid + WHERE objects.last_update IS NULL + OR updates.time_epoch_millis < :timeEpochMillisThreshold + ` + ) + .all({timeEpochMillisThreshold: +since}); + const connections: $PropertyType = db + .prepare( + dedent`\ + SELECT + connections.object_id AS objectId, + connections.fieldname AS fieldname, + connections.last_update IS NULL AS neverUpdated, + connections.end_cursor AS endCursor + FROM connections + LEFT OUTER JOIN updates ON connections.last_update = updates.rowid + WHERE connections.has_next_page + OR connections.last_update IS NULL + OR updates.time_epoch_millis < :timeEpochMillisThreshold + ` + ) + .all({timeEpochMillisThreshold: +since}) + .map((entry) => { + const result = {...entry}; + if (result.neverUpdated) { + result.endCursor = undefined; // as opposed to `null` + } + delete result.neverUpdated; + return result; + }); + return {objects, connections}; + }); + } } /** @@ -449,6 +501,29 @@ export function _buildSchemaInfo(schema: Schema.Schema): SchemaInfo { type UpdateId = number; +/** + * A set of objects and connections that should be updated. + */ +type QueryPlan = {| + +objects: $ReadOnlyArray<{| + +typename: Schema.Typename, + +id: Schema.ObjectId, + |}>, + +connections: $ReadOnlyArray<{| + +objectId: Schema.ObjectId, + +fieldname: Schema.Fieldname, + +endCursor: EndCursor | void, // `undefined` if never fetched + |}>, +|}; + +/** + * An `endCursor` of a GraphQL `pageInfo` object, denoting where the + * cursor should continue reading the next page. This is `null` when the + * cursor is at the beginning of the connection (i.e., when the + * connection is empty, or when `first: 0` is provided). + */ +type EndCursor = string | null; + /** * Execute a function inside a database transaction. * diff --git a/src/graphql/mirror.test.js b/src/graphql/mirror.test.js index 4be479d..9818fb1 100644 --- a/src/graphql/mirror.test.js +++ b/src/graphql/mirror.test.js @@ -4,6 +4,7 @@ import Database from "better-sqlite3"; import fs from "fs"; import tmp from "tmp"; +import dedent from "../util/dedent"; import * as Schema from "./schema"; import {_buildSchemaInfo, _inTransaction, Mirror} from "./mirror"; @@ -348,6 +349,137 @@ describe("graphql/mirror", () => { expect(db.prepare("SELECT * FROM connections").all()).toHaveLength(0); }); }); + + describe("_findOutdated", () => { + it("finds the right objects and connections", () => { + const db = new Database(":memory:"); + const schema = buildGithubSchema(); + const mirror = new Mirror(db, schema); + mirror.registerObject({typename: "Repository", id: "repo:ab/cd"}); + mirror.registerObject({typename: "Issue", id: "issue:ab/cd#1"}); + mirror.registerObject({typename: "Issue", id: "issue:ab/cd#2"}); + mirror.registerObject({typename: "Issue", id: "issue:ab/cd#3"}); + mirror.registerObject({typename: "Issue", id: "issue:ab/cd#4"}); + + const createUpdate = (epochTimeMillis) => ({ + time: epochTimeMillis, + id: mirror._createUpdate(new Date(epochTimeMillis)), + }); + const earlyUpdate = createUpdate(123); + const midUpdate = createUpdate(456); + const lateUpdate = createUpdate(789); + + const makeUpdateFunction = (updateSql) => { + const stmt = db.prepare(updateSql); + return (...bindings) => { + const result = stmt.run(...bindings); + // Make sure that we actually updated something. (This can + // trigger if, for instance, you copy-paste some updates for + // a new object, but never actually register that object + // with the DB.) + expect({updateSql, bindings, result}).toEqual({ + updateSql, + bindings, + result: expect.objectContaining({changes: 1}), + }); + }; + }; + + const setObjectData = makeUpdateFunction( + "UPDATE objects SET last_update = :update WHERE id = :id" + ); + setObjectData({id: "repo:ab/cd", update: earlyUpdate.id}); + setObjectData({id: "issue:ab/cd#1", update: lateUpdate.id}); + setObjectData({id: "issue:ab/cd#2", update: null}); + setObjectData({id: "issue:ab/cd#3", update: null}); + setObjectData({id: "issue:ab/cd#4", update: midUpdate.id}); + + const setConnectionData = makeUpdateFunction( + dedent`\ + UPDATE connections SET + last_update = :update, + total_count = :totalCount, + has_next_page = :hasNextPage, + end_cursor = :endCursor + WHERE object_id = :objectId AND fieldname = :fieldname + ` + ); + setConnectionData({ + objectId: "repo:ab/cd", + fieldname: "issues", + update: earlyUpdate.id, + totalCount: 1, + hasNextPage: +false, + endCursor: "cursor:repo.issues", + }); + setConnectionData({ + objectId: "issue:ab/cd#1", + fieldname: "comments", + update: null, + totalCount: null, + hasNextPage: null, + endCursor: null, + }); + setConnectionData({ + objectId: "issue:ab/cd#2", + fieldname: "comments", + update: lateUpdate.id, + totalCount: 1, + hasNextPage: +true, + endCursor: null, + }); + setConnectionData({ + objectId: "issue:ab/cd#3", + fieldname: "comments", + update: lateUpdate.id, + totalCount: 0, + hasNextPage: +false, + endCursor: null, + }); + setConnectionData({ + objectId: "issue:ab/cd#4", + fieldname: "comments", + update: midUpdate.id, + totalCount: 3, + hasNextPage: +false, + endCursor: "cursor:issue4.comments", + }); + + const actual = mirror._findOutdated(new Date(midUpdate.time)); + const expected = { + objects: [ + {typename: "Repository", id: "repo:ab/cd"}, // loaded before cutoff + // issue:ab/cd#1 was loaded after the cutoff + {typename: "Issue", id: "issue:ab/cd#2"}, // never loaded + {typename: "Issue", id: "issue:ab/cd#3"}, // never loaded + // issue:ab/cd#4 was loaded exactly at the cutoff + ], + connections: [ + { + // loaded before cutoff + objectId: "repo:ab/cd", + fieldname: "issues", + endCursor: "cursor:repo.issues", + }, + { + // never loaded + objectId: "issue:ab/cd#1", + fieldname: "comments", + endCursor: undefined, + }, + { + // loaded, but has more data available + objectId: "issue:ab/cd#2", + fieldname: "comments", + endCursor: null, + }, + // issue:ab/cd#3.comments was loaded after the cutoff + // issue:ab/cd#4.comments was loaded exactly at the cutoff + ], + }; + expect(actual).toEqual(expected); + }); + }); }); describe("_buildSchemaInfo", () => {