Github: switch to CacheProvider for fetchGithubRepo (#1614)

Delegating to a CacheProvider instance, will limit the number
of places where we need to handle filesystem details. It will
also allow a mock, or in-memory cache to be provided.
This commit is contained in:
Robin van Boven 2020-02-03 23:46:03 +01:00 committed by GitHub
parent 6904621646
commit ca63ea00fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 17 additions and 13 deletions

View File

@ -18,6 +18,8 @@ import * as Github from "../plugins/github/loadWeightedGraph";
import * as WeightedGraph from "../core/weightedGraph";
import {type Weights as WeightsT} from "../core/weights";
import {loadWeightedGraph} from "./loadWeightedGraph";
import {DataDirectory} from "../backend/dataDirectory";
import {type CacheProvider} from "../backend/cache";
export type LoadOptions = {|
+project: Project,
@ -58,6 +60,7 @@ export async function load(
const fullParams = params == null ? defaultParams() : partialParams(params);
const loadTask = `load-${options.project.id}`;
taskReporter.start(loadTask);
const dataDirectory = new DataDirectory(sourcecredDirectory);
const cacheDirectory = path.join(sourcecredDirectory, "cache");
await fs.mkdirp(cacheDirectory);
@ -77,7 +80,7 @@ export async function load(
githubOptions = {
repoIds: project.repoIds,
token: githubToken,
cacheDirectory,
cache: (dataDirectory: CacheProvider),
};
}

View File

@ -26,6 +26,7 @@ import {
partialParams,
} from "../analysis/timeline/params";
import * as WeightedGraph from "../core/weightedGraph";
import {DataDirectory} from "../backend/dataDirectory";
type JestMockFn = $Call<typeof jest.fn>;
jest.mock("../plugins/github/loadWeightedGraph", () => ({
@ -102,11 +103,11 @@ describe("api/load", () => {
it("calls github githubWeightedGraph with the right options", async () => {
const {options, taskReporter, sourcecredDirectory} = example();
await load(options, taskReporter);
const cacheDirectory = path.join(sourcecredDirectory, "cache");
const cache = new DataDirectory(sourcecredDirectory);
const expectedLoadGraphOptions: LoadGraphOptions = {
repoIds: project.repoIds,
token: exampleGithubToken,
cacheDirectory,
cache,
};
expect(githubWeightedGraph).toHaveBeenCalledWith(
expectedLoadGraphOptions,

View File

@ -13,10 +13,9 @@
*/
import stringify from "json-stable-stringify";
import tmp from "tmp";
import fetchGithubRepo from "../fetchGithubRepo";
import {makeRepoId} from "../repoId";
import {MemoryCacheProvider} from "../../../backend/memoryCacheProvider";
function parseArgs() {
const argv = process.argv.slice(2);
@ -38,7 +37,8 @@ function parseArgs() {
function main() {
const args = parseArgs();
const repoId = makeRepoId(args.owner, args.name);
const options = {token: args.githubToken, cacheDirectory: tmp.dirSync().name};
const cache = new MemoryCacheProvider();
const options = {token: args.githubToken, cache};
fetchGithubRepo(repoId, options)
.then((data) => {
console.log(stringify(data, {space: 4}));

View File

@ -6,7 +6,6 @@
import Database from "better-sqlite3";
import fetch from "isomorphic-fetch";
import path from "path";
import retry from "retry";
import {type RepoId, repoIdToString} from "./repoId";
@ -19,6 +18,7 @@ import type {Repository} from "./graphqlTypes";
import schema from "./schema";
import {validateToken} from "./token";
import {cacheIdForRepoId} from "./cacheId";
import {type CacheProvider} from "../../backend/cache";
/**
* Scrape data from a GitHub repo using the GitHub API.
@ -35,9 +35,9 @@ import {cacheIdForRepoId} from "./cacheId";
*/
export default async function fetchGithubRepo(
repoId: RepoId,
options: {|+token: string, +cacheDirectory: string|}
options: {|+token: string, +cache: CacheProvider|}
): Promise<Repository> {
const {token, cacheDirectory} = options;
const {token, cache} = options;
// Right now, only warn on likely to be bad tokens (see #1461).
// This lets us proceed to the GitHub API validating the token,
@ -59,8 +59,7 @@ export default async function fetchGithubRepo(
// name is valid and uniquely identifying even on case-insensitive
// filesystems (HFS, HFS+, APFS, NTFS) or filesystems preventing
// equals signs in file names.
const dbFilename = `${cacheIdForRepoId(repoId)}.db`;
const db = new Database(path.join(cacheDirectory, dbFilename));
const db: Database = await cache.database(cacheIdForRepoId(repoId));
const mirror = new Mirror(db, schema(), {
blacklistedIds: BLACKLISTED_IDS,
guessTypename: _guessTypename,

View File

@ -15,11 +15,12 @@ import {RelationalView} from "./relationalView";
import {type RepoId, repoIdToString} from "./repoId";
import {Graph} from "../../core/graph";
import {type GithubToken} from "./token";
import {type CacheProvider} from "../../backend/cache";
export type Options = {|
+repoIds: $ReadOnlyArray<RepoId>,
+token: GithubToken,
+cacheDirectory: string,
+cache: CacheProvider,
|};
/**
@ -42,7 +43,7 @@ export async function loadGraph(
repositories.push(
await fetchGithubRepo(repoId, {
token: options.token,
cacheDirectory: options.cacheDirectory,
cache: options.cache,
})
);
taskReporter.finish(taskId);