From 23ff0db5b5119965a9dbce4ce7b33212be87645d Mon Sep 17 00:00:00 2001 From: Marco Munizaga Date: Fri, 13 Jan 2023 10:23:58 -0800 Subject: [PATCH] Support other repos using the multidim interop tests (#99) * Support running interop workflow from other repos * A bit of shuffling and cleanup --- .github/actions/setup-testground/action.yml | 82 ------- .github/workflows/ping-interop-go.yml | 34 --- .github/workflows/ping-interop-latest.yml | 35 --- .github/workflows/ping-interop-rust.yml | 34 --- .github/workflows/ping-interop.yml | 45 ---- .github/workflows/run-composition.yml | 105 --------- .github/workflows/run-testplans.yml | 51 ++++- multidim-interop/go/v0.22/main.go | 2 +- multidim-interop/go/v0.23/main.go | 2 +- multidim-interop/go/v0.24/main.go | 2 +- multidim-interop/js/v0.41/index.js | 2 +- multidim-interop/package-lock.json | 226 +++++++++++++++++++- multidim-interop/package.json | 4 +- multidim-interop/testplans.ts | 66 +++++- 14 files changed, 333 insertions(+), 357 deletions(-) delete mode 100644 .github/actions/setup-testground/action.yml delete mode 100644 .github/workflows/ping-interop-go.yml delete mode 100644 .github/workflows/ping-interop-latest.yml delete mode 100644 .github/workflows/ping-interop-rust.yml delete mode 100644 .github/workflows/ping-interop.yml delete mode 100644 .github/workflows/run-composition.yml diff --git a/.github/actions/setup-testground/action.yml b/.github/actions/setup-testground/action.yml deleted file mode 100644 index 98b95dc..0000000 --- a/.github/actions/setup-testground/action.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: start testground -description: setup a local testground instance -inputs: - testground_endpoint: - required: false - default: '' - testground_repository: - required: false - default: 'testground/testground' - testground_ref: - required: false - default: 'edge' - -runs: - using: "composite" - steps: - # Default setup when we use the testground_ref == edge. - - name: Load testground - if: ${{ inputs.testground_ref == 'edge' }} - shell: bash - run: | - for i in 1 2 3; do - echo "=== Attempt $i ===" - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/testground/testground/master/install.sh)" && \ - exit 0 - sleep 10 - done - exit 1 - # Custom setup (slower) when we use a different testground_ref - - name: Checkout testground - if: ${{ inputs.testground_ref != 'edge' }} - uses: actions/checkout@v2 - with: - path: testground - repository: ${{ inputs.testground_repository }} - ref: ${{ inputs.testground_ref }} - - - name: Setup Go - if: ${{ inputs.testground_ref != 'edge' }} - uses: actions/setup-go@v3 - with: - cache: true - go-version-file: 'testground/go.mod' - cache-dependency-path: testground/go.sum - - - name: Install testground - if: ${{ inputs.testground_ref != 'edge' }} - run: make install || make install || make install # 3 retries in case of network drops. - working-directory: testground - shell: bash - - - name: Run the daemon or configure the client - shell: bash - env: - TESTGROUND_ENDPOINT: ${{ inputs.testground_endpoint }} - run: | - if [[ ! -z "${TESTGROUND_ENDPOINT}" ]]; then - mkdir -p ~/testground/; - cat <> ~/testground/.env.toml - - [client] - endpoint = "${TESTGROUND_ENDPOINT}" - EOF - else - mkdir -p ~/testground/; - cat <> ~/testground/.env.toml - - [daemon.scheduler] - task_timeout_min = 60 - EOF - testground daemon > testground.out 2> testground.err & - fi; - - - name: Check testground daemon health - run: - echo "Waiting for Testground to launch on 8042..."; - while ! nc -z localhost 8042; do - sleep 1; - done; - echo "Testground launched"; - testground healthcheck --runner local:docker --fix; - shell: bash \ No newline at end of file diff --git a/.github/workflows/ping-interop-go.yml b/.github/workflows/ping-interop-go.yml deleted file mode 100644 index 2282628..0000000 --- a/.github/workflows/ping-interop-go.yml +++ /dev/null @@ -1,34 +0,0 @@ -on: - workflow_dispatch: - inputs: - testground_endpoint: - type: string - required: false - description: testground endpoint - custom_git_reference: - type: string - required: false - description: go ref - custom_git_target: - type: string - required: false - description: repository to use as replace target - default: github.com/libp2p/go-libp2p - push: - pull_request: - -name: go-libp2p ping - go test with testground. - -jobs: - run-libp2p-test-plans: - # If you intend to use this workflow from another repo, - # you need to pass the repo and the version: - # uses: "libp2p/test-plans/.github/workflows/run-composition.yml@master" - uses: "./.github/workflows/run-composition.yml" - with: - composition_file: "ping/_compositions/go-cross-versions.toml" - custom_git_target: ${{ github.event.inputs.custom_git_target }} # nothing or "some-fork/go-libp2p" - custom_git_reference: ${{ github.event.inputs.custom_git_reference }} # a git branch - testground_endpoint: ${{ github.event.inputs.testground_endpoint }} - test_repository: ${{ (github.event.inputs && '') || github.repository }} - test_ref: ${{ (github.event.inputs && '') || github.event.pull_request.head.sha || github.sha }} \ No newline at end of file diff --git a/.github/workflows/ping-interop-latest.yml b/.github/workflows/ping-interop-latest.yml deleted file mode 100644 index 1f3c234..0000000 --- a/.github/workflows/ping-interop-latest.yml +++ /dev/null @@ -1,35 +0,0 @@ -on: - workflow_dispatch: - inputs: - testground_endpoint: - type: string - required: false - description: testground endpoint - custom_git_reference: - description: the git commit or branch we're going to use for the custom target - required: false - type: string - custom_git_target: - description: the custom git fork url we're going to use for the custom target (github.com/some-fork/rust-libp2p) - required: false - type: string - custom_interop_target: - description: in the case of cross-implementation testing, the implementation target (go | rust | nim) - required: false - type: string - push: - pull_request: - -name: libp2p ping - go, rust and nim test (latest) with testground. - -jobs: - run-ping-latest: - uses: "./.github/workflows/run-composition.yml" - with: - composition_file: "ping/_compositions/all-interop-latest.toml" - custom_git_target: ${{ github.event.inputs.custom_git_target }} # nothing or "some-fork/go-libp2p" - custom_git_reference: ${{ github.event.inputs.custom_git_reference }} # a git reference - custom_interop_target: ${{ github.event.inputs.custom_interop_target }} # go | rust | nim - testground_endpoint: ${{ github.event.inputs.testground_endpoint }} - test_repository: ${{ (github.event.inputs && '') || github.repository }} - test_ref: ${{ (github.event.inputs && '') || github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/ping-interop-rust.yml b/.github/workflows/ping-interop-rust.yml deleted file mode 100644 index 32b8f94..0000000 --- a/.github/workflows/ping-interop-rust.yml +++ /dev/null @@ -1,34 +0,0 @@ -on: - workflow_dispatch: - inputs: - testground_endpoint: - type: string - required: false - description: testground endpoint - custom_git_reference: - type: string - required: false - description: rust ref - custom_git_target: - type: string - required: false - description: repository to use as replace target - default: github.com/libp2p/rust-libp2p - push: - pull_request: - -name: rust-libp2p ping - rust test with testground. - -jobs: - run-libp2p-test-plans: - # If you intend to use this workflow from another repo, - # you need to pass the repo and the version: - # uses: "libp2p/test-plans/.github/workflows/run-composition.yml@master" - uses: "./.github/workflows/run-composition.yml" - with: - composition_file: "ping/_compositions/rust-cross-versions.toml" - custom_git_target: ${{ github.event.inputs.custom_git_target }} # nothing or "some-fork/go-libp2p" - custom_git_reference: ${{ github.event.inputs.custom_git_reference }} # a git reference - testground_endpoint: ${{ github.event.inputs.testground_endpoint }} - test_repository: ${{ (github.event.inputs && '') || github.repository }} - test_ref: ${{ (github.event.inputs && '') || github.event.pull_request.head.sha || github.sha }} \ No newline at end of file diff --git a/.github/workflows/ping-interop.yml b/.github/workflows/ping-interop.yml deleted file mode 100644 index 0f6b0fc..0000000 --- a/.github/workflows/ping-interop.yml +++ /dev/null @@ -1,45 +0,0 @@ -on: - workflow_dispatch: - inputs: - testground_endpoint: - type: string - required: false - description: testground endpoint - custom_git_reference: - description: the git commit or branch we're going to use for the custom target - required: false - type: string - custom_git_target: - description: the custom git fork url we're going to use for the custom target (github.com/some-fork/rust-libp2p) - required: false - type: string - custom_interop_target: - description: in the case of cross-implementation testing, the implementation target (go | rust | nim) - required: false - type: string - push: - pull_request: - -name: libp2p ping - go, rust and nim test (all) with testground. - -jobs: - run-ping-latest: - uses: "./.github/workflows/run-composition.yml" - with: - composition_file: "ping/_compositions/all-interop-latest.toml" - custom_git_target: ${{ github.event.inputs.custom_git_target }} # nothing or "some-fork/go-libp2p" - custom_git_reference: ${{ github.event.inputs.custom_git_reference }} # a git reference - custom_interop_target: ${{ github.event.inputs.custom_interop_target }} # go | rust | nim - testground_endpoint: ${{ github.event.inputs.testground_endpoint }} - test_repository: ${{ (github.event.inputs && '') || github.repository }} - test_ref: ${{ (github.event.inputs && '') || github.event.pull_request.head.sha || github.sha }} - run-ping-all: - uses: "./.github/workflows/run-composition.yml" - with: - composition_file: "ping/_compositions/all-interop.toml" - custom_git_target: ${{ github.event.inputs.custom_git_target }} # nothing or "some-fork/go-libp2p" - custom_git_reference: ${{ github.event.inputs.custom_git_reference }} # a git reference - custom_interop_target: ${{ github.event.inputs.custom_interop_target }} # go | rust | nim - testground_endpoint: ${{ github.event.inputs.testground_endpoint }} - test_repository: ${{ (github.event.inputs && '') || github.repository }} - test_ref: ${{ (github.event.inputs && '') || github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/run-composition.yml b/.github/workflows/run-composition.yml deleted file mode 100644 index 5b42e1a..0000000 --- a/.github/workflows/run-composition.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: Run composition file with a custom git reference - -on: - workflow_call: - inputs: - composition_file: - description: the workflow we're going to run - required: true - type: string - custom_git_reference: - description: the git commit or branch we're going to use for the custom target - required: false - type: string - custom_git_target: - description: the custom git fork url we're going to use for the custom target (github.com/some-fork/rust-libp2p) - required: false - type: string - custom_interop_target: - description: in the case of cross-implementation testing, the implementation target (go | rust | ...) - required: false - type: string - testground_endpoint: - required: false - type: string - test_repository: - required: false - type: string - test_ref: - required: false - type: string -jobs: - run_test: - name: Run a test with different versions - runs-on: ubuntu-latest - env: - TEST_PLAN_REPO: ${{ inputs.test_repository || 'libp2p/test-plans' }} - TEST_PLAN_BRANCH: ${{ inputs.test_ref || 'master' }} - TESTGROUND_ENDPOINT: ${{ inputs.testground_endpoint }} - COMPOSITION_FILE: ${{ inputs.composition_file }} - defaults: - run: - shell: bash - steps: - - name: Checkout sources - uses: actions/checkout@v2 - with: - path: test-plans - repository: ${{ env.TEST_PLAN_REPO }} - ref: ${{ env.TEST_PLAN_BRANCH }} - - name: setup testground - uses: ./test-plans/.github/actions/setup-testground - - name: Import the plan - working-directory: ./test-plans - run: | - testground plan import --from ./ --name libp2p - - name: Resolve the git references - if: ${{ inputs.custom_git_reference && inputs.custom_git_target }} - id: resolve_reference - working-directory: ./test-plans - run: | - git fetch https://${{ inputs.custom_git_target }} ${{ inputs.custom_git_reference }} - SHA=`git log FETCH_HEAD -n 1 --pretty=format:"%H"` - echo "::set-output name=custom_git_sha::${SHA}" - - name: Build the composition file - working-directory: ./test-plans - timeout-minutes: 40 - run: | - for i in 1 2 3; do - echo "=== Attempt $i ===" - testground build composition \ - -f "${COMPOSITION_FILE}" \ - --write-artifacts \ - --wait && exit 0; - sleep 10 - done - exit 1 - env: - GitReference: ${{ steps.resolve_reference.outputs.custom_git_sha || inputs.custom_git_reference }} - GitTarget: ${{ inputs.custom_git_target }} - InteropTarget: ${{ inputs.custom_interop_target }} - - name: Run the composition file - working-directory: ./test-plans - timeout-minutes: 6 - run: | - testground run composition \ - -f "${COMPOSITION_FILE}" \ - --metadata-repo "${GITHUB_REPOSITORY}" \ - --metadata-branch "${GITHUB_REF#refs/heads/}" \ - --metadata-commit "${GITHUB_SHA}" \ - --collect-file ./result.tgz \ - --collect --wait - env: - GitReference: ${{ steps.resolve_reference.outputs.custom_git_sha || inputs.custom_git_reference }} - GitTarget: ${{ inputs.custom_git_target }} - InteropTarget: ${{ inputs.custom_interop_target }} - - uses: actions/upload-artifact@v3 - if: ${{ failure() }} - with: - name: testground-output - path: | - ~/testground/ - ~/test-plans/result.tgz - ${{env.COMPOSITION_FILE}} - testground.* - test-plans/*.out diff --git a/.github/workflows/run-testplans.yml b/.github/workflows/run-testplans.yml index 638b1ec..505d7e1 100644 --- a/.github/workflows/run-testplans.yml +++ b/.github/workflows/run-testplans.yml @@ -7,6 +7,24 @@ on: description: the directory with the testplans test required: true type: string + extra-versions: + description: artifact name for the extra-versions.json file + required: false + type: string + image-tar: + description: artifact name for the image.tar(s) to import + required: false + type: string + test-filter: + description: "Filter which test runs" + type: string + required: false + default: "" + test-plans_ref: + description: "branch of test-plans to checkout" + type: string + required: false + default: "" jobs: run_test: name: Run testplans test @@ -18,15 +36,30 @@ jobs: shell: bash steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: test-plans - repository: ${{ env.TEST_PLAN_REPO }} - ref: ${{ env.TEST_PLAN_BRANCH }} + repository: "libp2p/test-plans" + ref: ${{ inputs.test-plans_ref }} + # Download input data + - uses: actions/download-artifact@v3 + if: ${{ inputs.extra-versions != '' }} + with: + name: ${{ inputs.extra-versions }} + path: /tmp/extra-versions + - uses: actions/download-artifact@v3 + if: ${{ inputs.image-tar != '' }} + with: + name: ${{ inputs.image-tar }} + path: /tmp/images/ + - name: Load docker images + if: ${{ inputs.image-tar != '' }} + run: for FILE in /tmp/images/*; do docker image load -i $FILE; done + # Continue with the test as before - uses: actions/setup-node@v3 with: node-version: 17 - cache: 'npm' + cache: "npm" cache-dependency-path: ./test-plans/${{ env.TEST_PLAN_DIR }}/package-lock.json - name: Expose GitHub Runtime # Needed for docker buildx to cache properly (See https://docs.docker.com/build/cache/backends/gha/#authentication) uses: crazy-max/ghaction-github-runtime@v2 @@ -39,13 +72,10 @@ jobs: - name: Build images working-directory: ./test-plans/${{ env.TEST_PLAN_DIR }}/ run: make - - name: List docker images - working-directory: ./test-plans/${{ env.TEST_PLAN_DIR }}/ - run: docker image ls - name: Run the test - timeout-minutes: 120 + timeout-minutes: 30 working-directory: ./test-plans/${{ env.TEST_PLAN_DIR }}/ - run: WORKER_COUNT=2 npm run test + run: WORKER_COUNT=2 npm run test -- --extra-versions-dir=/tmp/extra-versions --name-filter=${{ inputs.test-filter }} - name: Print the results working-directory: ./test-plans/${{ env.TEST_PLAN_DIR }}/ run: cat results.csv @@ -64,9 +94,8 @@ jobs: exit 0 fi - uses: actions/upload-artifact@v3 - if: ${{ always() }} with: - name: testground-output + name: test-plans-output path: | ./test-plans/${{ env.TEST_PLAN_DIR }}/results.csv ./test-plans/${{ env.TEST_PLAN_DIR }}/dashboard.md diff --git a/multidim-interop/go/v0.22/main.go b/multidim-interop/go/v0.22/main.go index eb42b62..8ab4f3a 100644 --- a/multidim-interop/go/v0.22/main.go +++ b/multidim-interop/go/v0.22/main.go @@ -137,7 +137,7 @@ func main() { if err != nil { panic("Failed to send listener address") } - _, err = rClient.BLPop(ctx, 4*time.Second, "dialerDone").Result() + _, err = rClient.BLPop(ctx, 10*time.Second, "dialerDone").Result() if err != nil { panic("Failed to wait for dialer conclusion") } diff --git a/multidim-interop/go/v0.23/main.go b/multidim-interop/go/v0.23/main.go index c5006a9..7768c6f 100644 --- a/multidim-interop/go/v0.23/main.go +++ b/multidim-interop/go/v0.23/main.go @@ -141,7 +141,7 @@ func main() { if err != nil { panic("Failed to send listener address") } - _, err = rClient.BLPop(ctx, 4*time.Second, "dialerDone").Result() + _, err = rClient.BLPop(ctx, 10*time.Second, "dialerDone").Result() if err != nil { panic("Failed to wait for dialer conclusion") } diff --git a/multidim-interop/go/v0.24/main.go b/multidim-interop/go/v0.24/main.go index 35fceab..ae1640f 100644 --- a/multidim-interop/go/v0.24/main.go +++ b/multidim-interop/go/v0.24/main.go @@ -145,7 +145,7 @@ func main() { if err != nil { panic("Failed to send listener address") } - _, err = rClient.BLPop(ctx, 4*time.Second, "dialerDone").Result() + _, err = rClient.BLPop(ctx, 10*time.Second, "dialerDone").Result() if err != nil { panic("Failed to wait for dialer conclusion") } diff --git a/multidim-interop/js/v0.41/index.js b/multidim-interop/js/v0.41/index.js index 45ee83d..dafc546 100644 --- a/multidim-interop/js/v0.41/index.js +++ b/multidim-interop/js/v0.41/index.js @@ -79,7 +79,7 @@ import { multiaddr } from '@multiformats/multiaddr' const multiaddrs = node.getMultiaddrs().map(ma => ma.toString()).filter(maString => !maString.includes("127.0.0.1")) console.log("My multiaddrs are", multiaddrs) await redisClient.rPush('listenerAddr', multiaddrs[0]) - await redisClient.blPop('dialerDone', 4) + await redisClient.blPop('dialerDone', 10) } try { diff --git a/multidim-interop/package-lock.json b/multidim-interop/package-lock.json index 12d9dc7..1c9ccf3 100644 --- a/multidim-interop/package-lock.json +++ b/multidim-interop/package-lock.json @@ -9,12 +9,14 @@ "version": "0.0.1", "license": "MIT", "dependencies": { + "@types/yargs": "^17.0.19", "csv-parse": "^5.3.3", "csv-stringify": "^6.2.3", "json-schema-to-typescript": "^11.0.2", "sqlite": "^4.1.2", "sqlite3": "^5.1.2", - "yaml": "^2.2.1" + "yaml": "^2.2.1", + "yargs": "^17.6.2" }, "devDependencies": { "ts-node": "^10.9.1", @@ -190,6 +192,19 @@ "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.2.tgz", "integrity": "sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==" }, + "node_modules/@types/yargs": { + "version": "17.0.19", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.19.tgz", + "integrity": "sha512-cAx3qamwaYX9R0fzOIZAlFpo4A+1uBVCxqpKz9D26uTF4srRXaGTTsikQmaotCtNdbhzyUH7ft6p9ktz9s6UNQ==", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", + "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==" + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -262,6 +277,20 @@ "node": ">=8" } }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -375,6 +404,35 @@ "node": ">=0.10" } }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "node_modules/color-support": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", @@ -538,6 +596,14 @@ "es6-symbol": "^3.1.1" } }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, "node_modules/event-emitter": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", @@ -595,6 +661,14 @@ "node": ">=10" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-stdin": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz", @@ -1279,6 +1353,14 @@ "node": ">= 6" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", @@ -1636,11 +1718,35 @@ "string-width": "^1.0.2 || 2 || 3 || 4" } }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -1654,6 +1760,31 @@ "node": ">= 14" } }, + "node_modules/yargs": { + "version": "17.6.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", + "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -1818,6 +1949,19 @@ "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.7.2.tgz", "integrity": "sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==" }, + "@types/yargs": { + "version": "17.0.19", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.19.tgz", + "integrity": "sha512-cAx3qamwaYX9R0fzOIZAlFpo4A+1uBVCxqpKz9D26uTF4srRXaGTTsikQmaotCtNdbhzyUH7ft6p9ktz9s6UNQ==", + "requires": { + "@types/yargs-parser": "*" + } + }, + "@types/yargs-parser": { + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", + "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==" + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -1869,6 +2013,14 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, "any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -1967,6 +2119,29 @@ "timers-ext": "^0.1.7" } }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "color-support": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", @@ -2103,6 +2278,11 @@ "es6-symbol": "^3.1.1" } }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, "event-emitter": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", @@ -2156,6 +2336,11 @@ "wide-align": "^1.1.2" } }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, "get-stdin": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-8.0.0.tgz", @@ -2677,6 +2862,11 @@ "util-deprecate": "^1.0.1" } }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" + }, "retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", @@ -2933,11 +3123,26 @@ "string-width": "^1.0.2 || 2 || 3 || 4" } }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, "yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -2948,6 +3153,25 @@ "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==" }, + "yargs": { + "version": "17.6.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", + "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" + }, "yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", diff --git a/multidim-interop/package.json b/multidim-interop/package.json index 98cd69a..1da0591 100644 --- a/multidim-interop/package.json +++ b/multidim-interop/package.json @@ -14,11 +14,13 @@ "typescript": "^4.9.3" }, "dependencies": { + "@types/yargs": "^17.0.19", "csv-parse": "^5.3.3", "csv-stringify": "^6.2.3", "json-schema-to-typescript": "^11.0.2", "sqlite": "^4.1.2", "sqlite3": "^5.1.2", - "yaml": "^2.2.1" + "yaml": "^2.2.1", + "yargs": "^17.6.2" } } diff --git a/multidim-interop/testplans.ts b/multidim-interop/testplans.ts index 1b7c47c..dd98e4e 100644 --- a/multidim-interop/testplans.ts +++ b/multidim-interop/testplans.ts @@ -1,12 +1,67 @@ import { buildTestSpecs } from "./src/generator" -import { versions } from './versions' -import { promises as fs } from 'fs'; +import { Version, versions } from "./versions" +import { promises as fs } from "fs"; import { run, RunFailure } from "./src/compose-runner" import { stringify } from "csv-stringify/sync" +import { stringify as YAMLStringify } from "yaml" +import yargs from "yargs/yargs" +import path from "path"; -const WorkerCount = parseInt(process.env.WORKER_COUNT || "1") +(async () => { + const WorkerCount = parseInt(process.env.WORKER_COUNT || "1") + const argv = await yargs(process.argv.slice(2)) + .options({ + 'name-filter': { + description: 'Only run named test', + default: "", + }, + 'emit-only': { + alias: 'e', + description: 'Only print the compose.yaml file', + default: false, + type: 'boolean' + }, + 'extra-versions-dir': { + description: 'Look for extra versions in this directory. Version files must be in json format', + default: "", + type: 'string' + }, + }) + .help() + .version(false) + .alias('help', 'h').argv; + const extraVersionsDir = argv.extraVersionsDir + const extraVersions: Array = [] + if (extraVersionsDir !== "") { + try { + const files = await fs.readdir(extraVersionsDir); + for (const file of files) { + const contents = await fs.readFile(path.join(extraVersionsDir, file)) + extraVersions.push(...JSON.parse(contents.toString())) + } + } catch (err) { + console.error("Error reading extra versions") + console.error(err); + } + } + + let testSpecs = await buildTestSpecs(versions.concat(extraVersions)) + + const nameFilter = argv["name-filter"] + if (nameFilter !== "") { + testSpecs = testSpecs.filter((testSpec) => testSpec.name?.includes(nameFilter)) + } + + + if (argv["emit-only"]) { + for (const testSpec of testSpecs) { + console.log("## " + testSpec.name) + console.log(YAMLStringify(testSpec)) + console.log("\n\n") + } + return + } -buildTestSpecs(versions).then(async (testSpecs) => { console.log(`Running ${testSpecs.length} tests`) const failures: Array = [] const statuses: Array = [["name", "outcome"]] @@ -31,4 +86,5 @@ buildTestSpecs(versions).then(async (testSpecs) => { console.log(`${failures.length} failures`, failures) await fs.writeFile("results.csv", stringify(statuses)) -}).then(() => console.log("Run complete")) \ No newline at end of file + console.log("Run complete") +})() \ No newline at end of file