132 lines
4.7 KiB
TypeScript
132 lines
4.7 KiB
TypeScript
import { buildTestSpecs } from "./src/generator"
|
|
import { Version, versions } from "./versions"
|
|
import { promises as fs } from "fs";
|
|
import {ExecException, run} from "./src/compose-runner"
|
|
import { stringify } from "csv-stringify/sync"
|
|
import { stringify as YAMLStringify } from "yaml"
|
|
import yargs from "yargs/yargs"
|
|
import path from "path";
|
|
|
|
(async () => {
|
|
const WorkerCount = parseInt(process.env.WORKER_COUNT || "1")
|
|
const argv = await yargs(process.argv.slice(2))
|
|
.options({
|
|
'name-filter': {
|
|
description: 'Only run tests including this name',
|
|
default: "",
|
|
},
|
|
'name-ignore': {
|
|
description: 'Do not run any tests including this name',
|
|
default: "",
|
|
},
|
|
'dry-run': {
|
|
description: "Don't actually run the test, just generate the compose files",
|
|
default: false,
|
|
type: 'boolean'
|
|
},
|
|
'extra-versions-dir': {
|
|
description: 'Look for extra versions in this directory. Version files must be in json format',
|
|
default: "",
|
|
type: 'string'
|
|
},
|
|
'extra-version': {
|
|
description: 'Paths to JSON files for additional versions to include in the test matrix',
|
|
default: [],
|
|
type: 'array'
|
|
},
|
|
})
|
|
.help()
|
|
.version(false)
|
|
.alias('help', 'h').argv;
|
|
const extraVersionsDir = argv.extraVersionsDir
|
|
const extraVersions: Array<Version> = []
|
|
if (extraVersionsDir !== "") {
|
|
try {
|
|
const files = await fs.readdir(extraVersionsDir);
|
|
for (const file of files) {
|
|
const contents = await fs.readFile(path.join(extraVersionsDir, file))
|
|
extraVersions.push(...JSON.parse(contents.toString()))
|
|
}
|
|
} catch (err) {
|
|
console.error("Error reading extra versions")
|
|
console.error(err);
|
|
}
|
|
}
|
|
|
|
for (let versionPath of argv.extraVersion.filter(p => p !== "")) {
|
|
const contents = await fs.readFile(versionPath);
|
|
extraVersions.push(JSON.parse(contents.toString()))
|
|
}
|
|
|
|
let nameFilter: string | null = argv["name-filter"]
|
|
if (nameFilter === "") {
|
|
nameFilter = null
|
|
}
|
|
let nameIgnore: string | null = argv["name-ignore"]
|
|
if (nameIgnore === "") {
|
|
nameIgnore = null
|
|
}
|
|
|
|
let routerImageId = JSON.parse(await fs.readFile(path.join(".", "router", "image.json"), "utf-8")).imageID;
|
|
let relayImageId = JSON.parse(await fs.readFile(path.join(".", "rust-relay", "image.json"), "utf-8")).imageID;
|
|
|
|
const routerDelay = 100;
|
|
const relayDelay = 25;
|
|
|
|
const rttRelayedConnection = routerDelay * 2 + relayDelay * 2;
|
|
const rttDirectConnection = routerDelay * 2;
|
|
|
|
const assetDir = path.join(__dirname, "runs");
|
|
|
|
let testSpecs = await buildTestSpecs(versions.concat(extraVersions), nameFilter, nameIgnore, routerImageId, relayImageId, routerDelay, relayDelay, assetDir)
|
|
|
|
console.log(`Running ${testSpecs.length} tests`)
|
|
const failures: Array<{ name: String, e: ExecException }> = []
|
|
const statuses: Array<string[]> = [["name", "outcome"]]
|
|
const workers = new Array(WorkerCount).fill({}).map(async () => {
|
|
while (true) {
|
|
const testSpec = testSpecs.pop()
|
|
if (testSpec == null) {
|
|
return
|
|
}
|
|
const name = testSpec.name;
|
|
if (!name) {
|
|
console.warn("Skipping testSpec without name")
|
|
continue;
|
|
}
|
|
|
|
console.log("Running test spec: " + name)
|
|
|
|
try {
|
|
const report = await run(testSpec, assetDir, argv['dry-run'] as boolean);
|
|
|
|
if (report != null) {
|
|
const rttDifference = Math.abs(report.rtt_to_holepunched_peer_millis - rttDirectConnection);
|
|
|
|
if (rttDifference > 5) {
|
|
// Emit a warning but don't do anything for now.
|
|
console.warn(`Expected RTT of direct connection to be ~${rttDirectConnection}ms but was ${report.rtt_to_holepunched_peer_millis}ms`)
|
|
}
|
|
}
|
|
|
|
statuses.push([name, "success"])
|
|
} catch (e) {
|
|
failures.push({ name, e })
|
|
statuses.push([name, "failure"])
|
|
}
|
|
}
|
|
})
|
|
await Promise.all(workers)
|
|
|
|
console.log(`${failures.length} failures:`)
|
|
|
|
for (const [number, {name, e}] of failures.entries()) {
|
|
console.log(`---------- ${name} ---------- (${number + 1} / ${failures.length})\n`);
|
|
console.log(e.stderr)
|
|
}
|
|
|
|
await fs.writeFile("results.csv", stringify(statuses))
|
|
|
|
console.log("Run complete")
|
|
})()
|