feat(hole-punch): add hole-punch interoperability test suite (#304)

.github/actions/run-interop-hole-punch-test/action.yml

@@ -0,0 +1,155 @@
+name: "libp2p hole-punch interop test"
+description: "Run the libp2p hole-punch interoperability test suite"
+inputs:
+  test-filter:
+    description: "Filter which tests to run out of the created matrix"
+    required: false
+    default: ""
+  test-ignore:
+    description: "Exclude tests from the created matrix that include this string in their name"
+    required: false
+    default: ""
+  extra-versions:
+    description: "Space-separated paths to JSON files describing additional images"
+    required: false
+    default: ""
+  s3-cache-bucket:
+    description: "Which S3 bucket to use for container layer caching"
+    required: false
+    default: ""
+  s3-access-key-id:
+    description: "S3 Access key id for the cache"
+    required: false
+    default: ""
+  s3-secret-access-key:
+    description: "S3 secret key id for the cache"
+    required: false
+    default: ""
+  aws-region:
+    description: "Which AWS region to use"
+    required: false
+    default: "us-east-1"
+  worker-count:
+    description: "How many workers to use for the test"
+    required: false
+    default: "2"
+runs:
+  using: "composite"
+  steps:
+    - name: Configure AWS credentials for S3 build cache
+      if: inputs.s3-access-key-id != '' && inputs.s3-secret-access-key != ''
+      run: |
+        echo "PUSH_CACHE=true" >> $GITHUB_ENV
+      shell: bash
+
+    # This depends on where this file is within this repository. This walks up
+    # from here to the hole-punch-interop folder
+    - run: |
+        WORK_DIR=$(realpath "$GITHUB_ACTION_PATH/../../../hole-punch-interop")
+        echo "WORK_DIR=$WORK_DIR" >> $GITHUB_OUTPUT
+      shell: bash
+      id: find-workdir
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 18
+
+    # Existence of /etc/buildkit/buildkitd.toml indicates that this is a
+    # self-hosted runner. If so, we need to pass the config to the buildx
+    # action. The config enables docker.io proxy which is required to
+    # work around docker hub rate limiting.
+    - run: |
+        if test -f /etc/buildkit/buildkitd.toml; then
+          echo "config=/etc/buildkit/buildkitd.toml" >> $GITHUB_OUTPUT
+        fi
+      shell: bash
+      id: buildkit
+
+    - name: Install more recent docker-compose version # https://stackoverflow.com/questions/54331949/having-networking-issues-with-docker-compose
+      shell: bash
+      run: |
+        mkdir -p $HOME/.docker/cli-plugins
+        wget -q -O- https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-linux-x86_64 > $HOME/.docker/cli-plugins/docker-compose
+        chmod +x $HOME/.docker/cli-plugins/docker-compose
+        docker compose version
+
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v2
+      with:
+        config: ${{ steps.buildkit.outputs.config }}
+
+    - name: Install deps
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      run: npm ci
+      shell: bash
+
+    - name: Load cache and build
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      run: npm run cache -- load
+      shell: bash
+
+    - name: Assert Git tree is clean.
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      shell: bash
+      run: |
+        if [[ -n "$(git status --porcelain)" ]]; then
+          echo "Git tree is dirty. This means that building an impl generated something that should probably be .gitignore'd"
+          git status
+          exit 1
+        fi
+
+    - name: Push the image cache
+      if: env.PUSH_CACHE == 'true'
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      env:
+        AWS_BUCKET: ${{ inputs.s3-cache-bucket }}
+        AWS_REGION: ${{ inputs.aws-region }}
+        AWS_ACCESS_KEY_ID: ${{ inputs.s3-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.s3-secret-access-key }}
+      run: npm run cache -- push
+      shell: bash
+
+    - name: Run the test
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      env:
+        WORKER_COUNT: ${{ inputs.worker-count }}
+        EXTRA_VERSION: ${{ inputs.extra-versions }}
+        NAME_FILTER: ${{ inputs.test-filter }}
+        NAME_IGNORE: ${{ inputs.test-ignore }}
+      run: npm run test -- --extra-version=$EXTRA_VERSION --name-filter=$NAME_FILTER --name-ignore=$NAME_IGNORE
+      shell: bash
+
+    - name: Print the results
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      run: cat results.csv
+      shell: bash
+
+    - name: Render results
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      run: npm run renderResults > ./dashboard.md
+      shell: bash
+
+    - name: Show Dashboard Output
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      run: cat ./dashboard.md >> $GITHUB_STEP_SUMMARY
+      shell: bash
+
+    - name: Exit with Error
+      working-directory: ${{ steps.find-workdir.outputs.WORK_DIR }}
+      run: |
+        if grep -q ":red_circle:" ./dashboard.md; then
+          exit 1
+        else
+          exit 0
+        fi
+      shell: bash
+
+    - uses: actions/upload-artifact@v3
+      if: ${{ always() }}
+      with:
+        name: test-plans-output
+        path: |
+          ${{ steps.find-workdir.outputs.WORK_DIR }}/results.csv
+          ${{ steps.find-workdir.outputs.WORK_DIR }}/dashboard.md
+          ${{ steps.find-workdir.outputs.WORK_DIR }}/runs

.github/workflows/hole-punch-interop.yml

@@ -0,0 +1,24 @@
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'hole-punch-interop/**'
+  push:
+    branches:
+      - "master"
+    paths:
+      - 'hole-punch-interop/**'
+
+name: libp2p holepunching interop test
+
+jobs:
+  run-hole-punch-interop:
+    runs-on: ['self-hosted', 'linux', 'x64', '4xlarge'] # https://github.com/pl-strflt/tf-aws-gh-runner/blob/main/runners.tf
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/run-interop-hole-punch-test
+        with:
+          s3-cache-bucket: libp2p-by-tf-aws-bootstrap
+          s3-access-key-id: ${{ vars.S3_AWS_ACCESS_KEY_ID }}
+          s3-secret-access-key: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
+          worker-count: 16

hole-punch-interop/.gitignore

@@ -0,0 +1,7 @@
+# For now, not committing image.json files
+image.json
+
+results.csv
+runs/
+
+node_modules/

hole-punch-interop/Makefile

@@ -0,0 +1,19 @@
+RUST_SUBDIRS := $(wildcard impl/rust/*/.)
+GO_SUBDIRS := $(wildcard impl/go/*/.)
+
+all: rust-relay router $(RUST_SUBDIRS) $(GO_SUBDIRS)
+rust-relay:
+	$(MAKE) -C rust-relay
+router:
+	$(MAKE) -C router
+$(RUST_SUBDIRS):
+	$(MAKE) -C $@
+$(GO_SUBDIRS):
+	$(MAKE) -C $@
+clean:
+	$(MAKE) -C rust-relay clean
+	$(MAKE) -C router clean
+	$(MAKE) -C $(RUST_SUBDIRS) clean
+	$(MAKE) -C $(GO_SUBDIRS) clean
+
+.PHONY: rust-relay router all $(RUST_SUBDIRS) $(GO_SUBDIRS)

hole-punch-interop/README.md

@@ -0,0 +1,84 @@
+# Hole punch tests
+
+## How to run locally
+
+1. `npm install`
+2. `make`
+3. `npm run test`
+
+## Client configuration
+
+| env variable | possible values |
+|--------------|-----------------|
+| MODE         | listen \| dial  |
+| TRANSPORT    | tcp \| quic     |
+
+- For TCP, the client MUST use noise + yamux to upgrade the connection.
+- The relayed connection MUST use noise + yamux.
+
+## Test flow
+
+1. The relay starts and pushes its address to the following redis keys:
+   - `RELAY_TCP_ADDRESS` for the TCP test
+   - `RELAY_QUIC_ADDRESS` for the QUIC test
+1. Upon start-up, clients connect to a redis server at `redis:6379` and block until this redis key comes available.
+   They then dial the relay on the provided address.
+1. The relay supports identify.
+   Implementations SHOULD use that to figure out their external address next.
+1. Once connected to the relay, a client in `MODE=listen` should listen on the relay and make a reservation.
+   Once the reservation is made, it pushes its `PeerId` to the redis key `LISTEN_CLIENT_PEER_ID`.
+1. A client in `MODE=dial` blocks on the availability of `LISTEN_CLIENT_PEER_ID`.
+   Once available, it dials `<relay_addr>/p2p-circuit/<listen-client-peer-id>`.
+1. Upon a successful hole-punch, the peer in `MODE=dial` measures the RTT across the newly established connection.
+1. The RTT MUST be printed to stdout in the following format:
+   ```json
+   { "rtt_to_holepunched_peer_millis": 12 }
+   ```
+1. Once printed, the dialer MUST exit with `0`.
+
+## Requirements for implementations
+
+- Docker containers MUST have a binary called `hole-punch-client` in their $PATH
+- MUST have `dig`, `curl`, `jq` and `tcpdump` installed
+- Listener MUST NOT early-exit but wait to be killed by test runner
+- Logs MUST go to stderr, RTT json MUST go to stdout
+- Dialer and lister both MUST use 0RTT negotiation for protocols
+- Implementations SHOULD disable timeouts on the redis client, i.e. use `0`
+- Implementations SHOULD exit early with a non-zero exit code if anything goes wrong
+- Implementations MUST set `TCP_NODELAY` for the TCP transport
+- Implements MUST make sure connections are being kept alive
+
+## Design notes
+
+The design of this test runner is heavily influenced by [multidim-interop](../multidim-interop) but differs in several ways.
+
+All files related to test runs will be written to the [./runs](./runs) directory.
+This includes the `docker-compose.yml` files of each individual run as well as logs and `tcpdump`'s for the dialer and listener.
+
+The docker-compose file uses 6 containers in total:
+
+- 1 redis container for orchestrating the test
+- 1 [relay](./rust-relay)
+- 1 hole-punch client in `MODE=dial`
+- 1 hole-punch client in `MODE=listen`
+- 2 [routers](./router): 1 per client
+
+The networks are allocated by docker-compose.
+We dynamically fetch the IPs and subnets as part of a startup script to set the correct IP routes.  
+
+In total, we have three networks:
+
+1. `lan_dialer`
+2. `lan_listener`
+3. `internet`
+
+The two LANs host a router and a client each whereas the relay is connected (without a router) to the `internet` network.
+On startup of the clients, we add an `ip route` that redirects all traffic to the corresponding `router` container.
+The router container masquerades all traffic upon forwarding, see the [README](./router/README.md) for details.
+
+## Running a single test
+
+1. Build all containers using `make`
+1. Generate all test definitions using `npm run test -- --no-run`
+1. Pick the desired test from the [runs](./runs) directory
+1. Execute it using `docker compose up`

hole-punch-interop/compose-spec/compose-spec.json

@@ -0,0 +1,818 @@
+{
+  "$schema": "http://json-schema.org/draft/2019-09/schema#",
+  "id": "compose_spec.json",
+  "type": "object",
+  "title": "Compose Specification",
+  "description": "The Compose file is a YAML file defining a multi-containers based application.",
+
+  "properties": {
+    "version": {
+      "type": "string",
+      "description": "declared for backward compatibility, ignored."
+    },
+
+    "name": {
+      "type": "string",
+      "description": "define the Compose project name, until user defines one explicitly."
+    },
+
+    "services": {
+      "id": "#/properties/services",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/service"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "networks": {
+      "id": "#/properties/networks",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/network"
+        }
+      }
+    },
+
+    "volumes": {
+      "id": "#/properties/volumes",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/volume"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "secrets": {
+      "id": "#/properties/secrets",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/secret"
+        }
+      },
+      "additionalProperties": false
+    },
+
+    "configs": {
+      "id": "#/properties/configs",
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9._-]+$": {
+          "$ref": "#/definitions/config"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+
+  "patternProperties": {"^x-": {}},
+  "additionalProperties": false,
+
+  "definitions": {
+
+    "service": {
+      "id": "#/definitions/service",
+      "type": "object",
+
+      "properties": {
+        "deploy": {"$ref": "#/definitions/deployment"},
+        "build": {
+          "oneOf": [
+            {"type": "string"},
+            {
+              "type": "object",
+              "properties": {
+                "context": {"type": "string"},
+                "dockerfile": {"type": "string"},
+                "args": {"$ref": "#/definitions/list_or_dict"},
+                "ssh": {"$ref": "#/definitions/list_or_dict"},
+                "labels": {"$ref": "#/definitions/list_or_dict"},
+                "cache_from": {"type": "array", "items": {"type": "string"}},
+                "cache_to": {"type": "array", "items": {"type": "string"}},
+                "no_cache": {"type": "boolean"},
+                "network": {"type": "string"},
+                "pull": {"type": "boolean"},
+                "target": {"type": "string"},
+                "shm_size": {"type": ["integer", "string"]},
+                "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
+                "isolation": {"type": "string"},
+                "privileged": {"type": "boolean"},
+                "secrets": {"$ref": "#/definitions/service_config_or_secret"},
+                "tags": {"type": "array", "items": {"type": "string"}},
+                "platforms": {"type": "array", "items": {"type": "string"}}
+              },
+              "additionalProperties": false,
+              "patternProperties": {"^x-": {}}
+            }
+          ]
+        },
+        "blkio_config": {
+          "type": "object",
+          "properties": {
+            "device_read_bps": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_limit"}
+            },
+            "device_read_iops": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_limit"}
+            },
+            "device_write_bps": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_limit"}
+            },
+            "device_write_iops": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_limit"}
+            },
+            "weight": {"type": "integer"},
+            "weight_device": {
+              "type": "array",
+              "items": {"$ref": "#/definitions/blkio_weight"}
+            }
+          },
+          "additionalProperties": false
+        },
+        "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "cgroup": {"type": "string", "enum": ["host", "private"]},
+        "cgroup_parent": {"type": "string"},
+        "command": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "configs": {"$ref": "#/definitions/service_config_or_secret"},
+        "container_name": {"type": "string"},
+        "cpu_count": {"type": "integer", "minimum": 0},
+        "cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100},
+        "cpu_shares": {"type": ["number", "string"]},
+        "cpu_quota": {"type": ["number", "string"]},
+        "cpu_period": {"type": ["number", "string"]},
+        "cpu_rt_period": {"type": ["number", "string"]},
+        "cpu_rt_runtime": {"type": ["number", "string"]},
+        "cpus": {"type": ["number", "string"]},
+        "cpuset": {"type": "string"},
+        "credential_spec": {
+          "type": "object",
+          "properties": {
+            "config": {"type": "string"},
+            "file": {"type": "string"},
+            "registry": {"type": "string"}
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "depends_on": {
+          "oneOf": [
+            {"$ref": "#/definitions/list_of_strings"},
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "patternProperties": {
+                "^[a-zA-Z0-9._-]+$": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "properties": {
+                    "condition": {
+                      "type": "string",
+                      "enum": ["service_started", "service_healthy", "service_completed_successfully"]
+                    }
+                  },
+                  "required": ["condition"]
+                }
+              }
+            }
+          ]
+        },
+        "device_cgroup_rules": {"$ref": "#/definitions/list_of_strings"},
+        "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "dns": {"$ref": "#/definitions/string_or_list"},
+        "dns_opt": {"type": "array","items": {"type": "string"}, "uniqueItems": true},
+        "dns_search": {"$ref": "#/definitions/string_or_list"},
+        "domainname": {"type": "string"},
+        "entrypoint": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "env_file": {"$ref": "#/definitions/string_or_list"},
+        "environment": {"$ref": "#/definitions/list_or_dict"},
+
+        "expose": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"],
+            "format": "expose"
+          },
+          "uniqueItems": true
+        },
+        "extends": {
+          "oneOf": [
+            {"type": "string"},
+            {
+              "type": "object",
+
+              "properties": {
+                "service": {"type": "string"},
+                "file": {"type": "string"}
+              },
+              "required": ["service"],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "extra_hosts": {"$ref": "#/definitions/list_or_dict"},
+        "group_add": {
+          "type": "array",
+          "items": {
+            "type": ["string", "number"]
+          },
+          "uniqueItems": true
+        },
+        "healthcheck": {"$ref": "#/definitions/healthcheck"},
+        "hostname": {"type": "string"},
+        "image": {"type": "string"},
+        "init": {"type": "boolean"},
+        "ipc": {"type": "string"},
+        "isolation": {"type": "string"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "logging": {
+          "type": "object",
+
+          "properties": {
+            "driver": {"type": "string"},
+            "options": {
+              "type": "object",
+              "patternProperties": {
+                "^.+$": {"type": ["string", "number", "null"]}
+              }
+            }
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "mac_address": {"type": "string"},
+        "mem_limit": {"type": ["number", "string"]},
+        "mem_reservation": {"type": ["string", "integer"]},
+        "mem_swappiness": {"type": "integer"},
+        "memswap_limit": {"type": ["number", "string"]},
+        "network_mode": {"type": "string"},
+        "networks": {
+          "oneOf": [
+            {"$ref": "#/definitions/list_of_strings"},
+            {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9._-]+$": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "properties": {
+                        "aliases": {"$ref": "#/definitions/list_of_strings"},
+                        "ipv4_address": {"type": "string"},
+                        "ipv6_address": {"type": "string"},
+                        "link_local_ips": {"$ref": "#/definitions/list_of_strings"},
+                        "priority": {"type": "number"}
+                      },
+                      "additionalProperties": false,
+                      "patternProperties": {"^x-": {}}
+                    },
+                    {"type": "null"}
+                  ]
+                }
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "oom_kill_disable": {"type": "boolean"},
+        "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000},
+        "pid": {"type": ["string", "null"]},
+        "pids_limit": {"type": ["number", "string"]},
+        "platform": {"type": "string"},
+        "ports": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": "number", "format": "ports"},
+              {"type": "string", "format": "ports"},
+              {
+                "type": "object",
+                "properties": {
+                  "mode": {"type": "string"},
+                  "host_ip": {"type": "string"},
+                  "target": {"type": "integer"},
+                  "published": {"type": ["string", "integer"]},
+                  "protocol": {"type": "string"}
+                },
+                "additionalProperties": false,
+                "patternProperties": {"^x-": {}}
+              }
+            ]
+          },
+          "uniqueItems": true
+        },
+        "privileged": {"type": "boolean"},
+        "profiles": {"$ref": "#/definitions/list_of_strings"},
+        "pull_policy": {"type": "string", "enum": [
+          "always", "never", "if_not_present", "build", "missing"
+        ]},
+        "read_only": {"type": "boolean"},
+        "restart": {"type": "string"},
+        "runtime": {
+          "type": "string"
+        },
+        "scale": {
+          "type": "integer"
+        },
+        "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
+        "shm_size": {"type": ["number", "string"]},
+        "secrets": {"$ref": "#/definitions/service_config_or_secret"},
+        "sysctls": {"$ref": "#/definitions/list_or_dict"},
+        "stdin_open": {"type": "boolean"},
+        "stop_grace_period": {"type": "string", "format": "duration"},
+        "stop_signal": {"type": "string"},
+        "storage_opt": {"type": "object"},
+        "tmpfs": {"$ref": "#/definitions/string_or_list"},
+        "tty": {"type": "boolean"},
+        "ulimits": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-z]+$": {
+              "oneOf": [
+                {"type": "integer"},
+                {
+                  "type": "object",
+                  "properties": {
+                    "hard": {"type": "integer"},
+                    "soft": {"type": "integer"}
+                  },
+                  "required": ["soft", "hard"],
+                  "additionalProperties": false,
+                  "patternProperties": {"^x-": {}}
+                }
+              ]
+            }
+          }
+        },
+        "user": {"type": "string"},
+        "uts": {"type": "string"},
+        "userns_mode": {"type": "string"},
+        "volumes": {
+          "type": "array",
+          "items": {
+            "oneOf": [
+              {"type": "string"},
+              {
+                "type": "object",
+                "required": ["type"],
+                "properties": {
+                  "type": {"type": "string"},
+                  "source": {"type": "string"},
+                  "target": {"type": "string"},
+                  "read_only": {"type": "boolean"},
+                  "consistency": {"type": "string"},
+                  "bind": {
+                    "type": "object",
+                    "properties": {
+                      "propagation": {"type": "string"},
+                      "create_host_path": {"type": "boolean"},
+                      "selinux": {"type": "string", "enum": ["z", "Z"]}
+                    },
+                    "additionalProperties": false,
+                    "patternProperties": {"^x-": {}}
+                  },
+                  "volume": {
+                    "type": "object",
+                    "properties": {
+                      "nocopy": {"type": "boolean"}
+                    },
+                    "additionalProperties": false,
+                    "patternProperties": {"^x-": {}}
+                  },
+                  "tmpfs": {
+                    "type": "object",
+                    "properties": {
+                      "size": {
+                        "oneOf": [
+                          {"type": "integer", "minimum": 0},
+                          {"type": "string"}
+                        ]
+                      },
+                      "mode": {"type": "number"}
+                    },
+                    "additionalProperties": false,
+                    "patternProperties": {"^x-": {}}
+                  }
+                },
+                "additionalProperties": false,
+                "patternProperties": {"^x-": {}}
+              }
+            ]
+          },
+          "uniqueItems": true
+        },
+        "volumes_from": {
+          "type": "array",
+          "items": {"type": "string"},
+          "uniqueItems": true
+        },
+        "working_dir": {"type": "string"}
+      },
+      "patternProperties": {"^x-": {}},
+      "additionalProperties": false
+    },
+
+    "healthcheck": {
+      "id": "#/definitions/healthcheck",
+      "type": "object",
+      "properties": {
+        "disable": {"type": "boolean"},
+        "interval": {"type": "string", "format": "duration"},
+        "retries": {"type": "number"},
+        "test": {
+          "oneOf": [
+            {"type": "string"},
+            {"type": "array", "items": {"type": "string"}}
+          ]
+        },
+        "timeout": {"type": "string", "format": "duration"},
+        "start_period": {"type": "string", "format": "duration"}
+      },
+      "additionalProperties": false,
+      "patternProperties": {"^x-": {}}
+    },
+    "deployment": {
+      "id": "#/definitions/deployment",
+      "type": ["object", "null"],
+      "properties": {
+        "mode": {"type": "string"},
+        "endpoint_mode": {"type": "string"},
+        "replicas": {"type": "integer"},
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "rollback_config": {
+          "type": "object",
+          "properties": {
+            "parallelism": {"type": "integer"},
+            "delay": {"type": "string", "format": "duration"},
+            "failure_action": {"type": "string"},
+            "monitor": {"type": "string", "format": "duration"},
+            "max_failure_ratio": {"type": "number"},
+            "order": {"type": "string", "enum": [
+              "start-first", "stop-first"
+            ]}
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "update_config": {
+          "type": "object",
+          "properties": {
+            "parallelism": {"type": "integer"},
+            "delay": {"type": "string", "format": "duration"},
+            "failure_action": {"type": "string"},
+            "monitor": {"type": "string", "format": "duration"},
+            "max_failure_ratio": {"type": "number"},
+            "order": {"type": "string", "enum": [
+              "start-first", "stop-first"
+            ]}
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "resources": {
+          "type": "object",
+          "properties": {
+            "limits": {
+              "type": "object",
+              "properties": {
+                "cpus": {"type": ["number", "string"]},
+                "memory": {"type": "string"},
+                "pids": {"type": "integer"}
+              },
+              "additionalProperties": false,
+              "patternProperties": {"^x-": {}}
+            },
+            "reservations": {
+              "type": "object",
+              "properties": {
+                "cpus": {"type": ["number", "string"]},
+                "memory": {"type": "string"},
+                "generic_resources": {"$ref": "#/definitions/generic_resources"},
+                "devices": {"$ref": "#/definitions/devices"}
+              },
+              "additionalProperties": false,
+              "patternProperties": {"^x-": {}}
+            }
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "restart_policy": {
+          "type": "object",
+          "properties": {
+            "condition": {"type": "string"},
+            "delay": {"type": "string", "format": "duration"},
+            "max_attempts": {"type": "integer"},
+            "window": {"type": "string", "format": "duration"}
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "placement": {
+          "type": "object",
+          "properties": {
+            "constraints": {"type": "array", "items": {"type": "string"}},
+            "preferences": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "spread": {"type": "string"}
+                },
+                "additionalProperties": false,
+                "patternProperties": {"^x-": {}}
+              }
+            },
+            "max_replicas_per_node": {"type": "integer"}
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        }
+      },
+      "additionalProperties": false,
+      "patternProperties": {"^x-": {}}
+    },
+
+    "generic_resources": {
+      "id": "#/definitions/generic_resources",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "discrete_resource_spec": {
+            "type": "object",
+            "properties": {
+              "kind": {"type": "string"},
+              "value": {"type": "number"}
+            },
+            "additionalProperties": false,
+            "patternProperties": {"^x-": {}}
+          }
+        },
+        "additionalProperties": false,
+        "patternProperties": {"^x-": {}}
+      }
+    },
+
+    "devices": {
+      "id": "#/definitions/devices",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+            "capabilities": {"$ref": "#/definitions/list_of_strings"},
+            "count": {"type": ["string", "integer"]},
+            "device_ids": {"$ref": "#/definitions/list_of_strings"},
+            "driver":{"type": "string"},
+            "options":{"$ref": "#/definitions/list_or_dict"}
+          },
+        "additionalProperties": false,
+        "patternProperties": {"^x-": {}}
+      }
+    },
+
+    "network": {
+      "id": "#/definitions/network",
+      "type": ["object", "null"],
+      "properties": {
+        "name": {"type": "string"},
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "ipam": {
+          "type": "object",
+          "properties": {
+            "driver": {"type": "string"},
+            "config": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "subnet": {"type": "string", "format": "subnet_ip_address"},
+                  "ip_range": {"type": "string"},
+                  "gateway": {"type": "string"},
+                  "aux_addresses": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "patternProperties": {"^.+$": {"type": "string"}}
+                  }
+                },
+                "additionalProperties": false,
+                "patternProperties": {"^x-": {}}
+              }
+            },
+            "options": {
+              "type": "object",
+              "additionalProperties": false,
+              "patternProperties": {"^.+$": {"type": "string"}}
+            }
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {
+              "deprecated": true,
+              "type": "string"
+            }
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "internal": {"type": "boolean"},
+        "enable_ipv6": {"type": "boolean"},
+        "attachable": {"type": "boolean"},
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false,
+      "patternProperties": {"^x-": {}}
+    },
+
+    "volume": {
+      "id": "#/definitions/volume",
+      "type": ["object", "null"],
+      "properties": {
+        "name": {"type": "string"},
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {
+              "deprecated": true,
+              "type": "string"
+            }
+          },
+          "additionalProperties": false,
+          "patternProperties": {"^x-": {}}
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"}
+      },
+      "additionalProperties": false,
+      "patternProperties": {"^x-": {}}
+    },
+
+    "secret": {
+      "id": "#/definitions/secret",
+      "type": "object",
+      "properties": {
+        "name": {"type": "string"},
+        "environment": {"type": "string"},
+        "file": {"type": "string"},
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {"type": "string"}
+          }
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "driver": {"type": "string"},
+        "driver_opts": {
+          "type": "object",
+          "patternProperties": {
+            "^.+$": {"type": ["string", "number"]}
+          }
+        },
+        "template_driver": {"type": "string"}
+      },
+      "additionalProperties": false,
+      "patternProperties": {"^x-": {}}
+    },
+
+    "config": {
+      "id": "#/definitions/config",
+      "type": "object",
+      "properties": {
+        "name": {"type": "string"},
+        "file": {"type": "string"},
+        "external": {
+          "type": ["boolean", "object"],
+          "properties": {
+            "name": {
+              "deprecated": true,
+              "type": "string"
+            }
+          }
+        },
+        "labels": {"$ref": "#/definitions/list_or_dict"},
+        "template_driver": {"type": "string"}
+      },
+      "additionalProperties": false,
+      "patternProperties": {"^x-": {}}
+    },
+
+    "string_or_list": {
+      "oneOf": [
+        {"type": "string"},
+        {"$ref": "#/definitions/list_of_strings"}
+      ]
+    },
+
+    "list_of_strings": {
+      "type": "array",
+      "items": {"type": "string"},
+      "uniqueItems": true
+    },
+
+    "list_or_dict": {
+      "oneOf": [
+        {
+          "type": "object",
+          "patternProperties": {
+            ".+": {
+              "type": ["string", "number", "boolean", "null"]
+            }
+          },
+          "additionalProperties": false
+        },
+        {"type": "array", "items": {"type": "string"}, "uniqueItems": true}
+      ]
+    },
+
+    "blkio_limit": {
+      "type": "object",
+      "properties": {
+        "path": {"type": "string"},
+        "rate": {"type": ["integer", "string"]}
+      },
+      "additionalProperties": false
+    },
+    "blkio_weight": {
+      "type": "object",
+      "properties": {
+        "path": {"type": "string"},
+        "weight": {"type": "integer"}
+      },
+      "additionalProperties": false
+    },
+
+    "service_config_or_secret": {
+      "type": "array",
+      "items": {
+        "oneOf": [
+          {"type": "string"},
+          {
+            "type": "object",
+            "properties": {
+              "source": {"type": "string"},
+              "target": {"type": "string"},
+              "uid": {"type": "string"},
+              "gid": {"type": "string"},
+              "mode": {"type": "number"}
+            },
+            "additionalProperties": false,
+            "patternProperties": {"^x-": {}}
+          }
+        ]
+      }
+    },
+
+    "constraints": {
+      "service": {
+        "id": "#/definitions/constraints/service",
+        "anyOf": [
+          {"required": ["build"]},
+          {"required": ["image"]}
+        ],
+        "properties": {
+          "build": {
+            "required": ["context"]
+          }
+        }
+      }
+    }
+  }
+}

hole-punch-interop/compose-spec/compose-spec.ts

@@ -0,0 +1,818 @@
+/* tslint:disable */
+/**
+ * This file was automatically generated by json-schema-to-typescript.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run json-schema-to-typescript to regenerate this file.
+ */
+
+export type DefinitionsDeployment = {
+  mode?: string;
+  endpoint_mode?: string;
+  replicas?: number;
+  labels?: ListOrDict;
+  rollback_config?: {
+    parallelism?: number;
+    delay?: string;
+    failure_action?: string;
+    monitor?: string;
+    max_failure_ratio?: number;
+    order?: "start-first" | "stop-first";
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  update_config?: {
+    parallelism?: number;
+    delay?: string;
+    failure_action?: string;
+    monitor?: string;
+    max_failure_ratio?: number;
+    order?: "start-first" | "stop-first";
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  resources?: {
+    limits?: {
+      cpus?: number | string;
+      memory?: string;
+      pids?: number;
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    };
+    reservations?: {
+      cpus?: number | string;
+      memory?: string;
+      generic_resources?: DefinitionsGenericResources;
+      devices?: DefinitionsDevices;
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    };
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  restart_policy?: {
+    condition?: string;
+    delay?: string;
+    max_attempts?: number;
+    window?: string;
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  placement?: {
+    constraints?: string[];
+    preferences?: {
+      spread?: string;
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    }[];
+    max_replicas_per_node?: number;
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  /**
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   *
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+} & Deployment;
+export type ListOrDict =
+  | {
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` ".+".
+       */
+      [k: string]: string | number | boolean | null;
+    }
+  | string[];
+export type DefinitionsGenericResources = {
+  discrete_resource_spec?: {
+    kind?: string;
+    value?: number;
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  /**
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+}[];
+export type ListOfStrings = string[];
+export type DefinitionsDevices = {
+  capabilities?: ListOfStrings;
+  count?: string | number;
+  device_ids?: ListOfStrings;
+  driver?: string;
+  options?: ListOrDict;
+  /**
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+}[];
+export type Deployment = {
+  mode?: string;
+  endpoint_mode?: string;
+  replicas?: number;
+  labels?: ListOrDict;
+  rollback_config?: {
+    parallelism?: number;
+    delay?: string;
+    failure_action?: string;
+    monitor?: string;
+    max_failure_ratio?: number;
+    order?: "start-first" | "stop-first";
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  update_config?: {
+    parallelism?: number;
+    delay?: string;
+    failure_action?: string;
+    monitor?: string;
+    max_failure_ratio?: number;
+    order?: "start-first" | "stop-first";
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  resources?: {
+    limits?: {
+      cpus?: number | string;
+      memory?: string;
+      pids?: number;
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    };
+    reservations?: {
+      cpus?: number | string;
+      memory?: string;
+      generic_resources?: DefinitionsGenericResources;
+      devices?: DefinitionsDevices;
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    };
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  restart_policy?: {
+    condition?: string;
+    delay?: string;
+    max_attempts?: number;
+    window?: string;
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  placement?: {
+    constraints?: string[];
+    preferences?: {
+      spread?: string;
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    }[];
+    max_replicas_per_node?: number;
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  /**
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   *
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+} | null;
+export type ServiceConfigOrSecret = (
+  | string
+  | {
+      source?: string;
+      target?: string;
+      uid?: string;
+      gid?: string;
+      mode?: number;
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    }
+)[];
+export type StringOrList = string | ListOfStrings;
+/**
+ * This interface was referenced by `PropertiesNetworks`'s JSON-Schema definition
+ * via the `patternProperty` "^[a-zA-Z0-9._-]+$".
+ */
+export type DefinitionsNetwork = {
+  name?: string;
+  driver?: string;
+  driver_opts?: {
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^.+$".
+     */
+    [k: string]: string | number;
+  };
+  ipam?: {
+    driver?: string;
+    config?: {
+      subnet?: string;
+      ip_range?: string;
+      gateway?: string;
+      aux_addresses?: {
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^.+$".
+         */
+        [k: string]: string;
+      };
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    }[];
+    options?: {
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^.+$".
+       */
+      [k: string]: string;
+    };
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  external?:
+    | boolean
+    | {
+        name?: string;
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^x-".
+         */
+        [k: string]: unknown;
+      };
+  internal?: boolean;
+  enable_ipv6?: boolean;
+  attachable?: boolean;
+  labels?: ListOrDict;
+  /**
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   *
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+} & Network;
+export type Network = {
+  name?: string;
+  driver?: string;
+  driver_opts?: {
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^.+$".
+     */
+    [k: string]: string | number;
+  };
+  ipam?: {
+    driver?: string;
+    config?: {
+      subnet?: string;
+      ip_range?: string;
+      gateway?: string;
+      aux_addresses?: {
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^.+$".
+         */
+        [k: string]: string;
+      };
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^x-".
+       */
+      [k: string]: unknown;
+    }[];
+    options?: {
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^.+$".
+       */
+      [k: string]: string;
+    };
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  external?:
+    | boolean
+    | {
+        name?: string;
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^x-".
+         */
+        [k: string]: unknown;
+      };
+  internal?: boolean;
+  enable_ipv6?: boolean;
+  attachable?: boolean;
+  labels?: ListOrDict;
+  /**
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   *
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+} | null;
+/**
+ * This interface was referenced by `PropertiesVolumes`'s JSON-Schema definition
+ * via the `patternProperty` "^[a-zA-Z0-9._-]+$".
+ */
+export type DefinitionsVolume = {
+  name?: string;
+  driver?: string;
+  driver_opts?: {
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^.+$".
+     */
+    [k: string]: string | number;
+  };
+  external?:
+    | boolean
+    | {
+        name?: string;
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^x-".
+         */
+        [k: string]: unknown;
+      };
+  labels?: ListOrDict;
+  /**
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   *
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+} & Volume;
+export type Volume = {
+  name?: string;
+  driver?: string;
+  driver_opts?: {
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^.+$".
+     */
+    [k: string]: string | number;
+  };
+  external?:
+    | boolean
+    | {
+        name?: string;
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^x-".
+         */
+        [k: string]: unknown;
+      };
+  labels?: ListOrDict;
+  /**
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   *
+   * This interface was referenced by `undefined`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+} | null;
+
+/**
+ * The Compose file is a YAML file defining a multi-containers based application.
+ */
+export interface ComposeSpecification {
+  /**
+   * declared for backward compatibility, ignored.
+   */
+  version?: string;
+  /**
+   * define the Compose project name, until user defines one explicitly.
+   */
+  name?: string;
+  services?: PropertiesServices;
+  networks?: PropertiesNetworks;
+  volumes?: PropertiesVolumes;
+  secrets?: PropertiesSecrets;
+  configs?: PropertiesConfigs;
+  /**
+   * This interface was referenced by `ComposeSpecification`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+}
+export interface PropertiesServices {
+  [k: string]: DefinitionsService;
+}
+/**
+ * This interface was referenced by `PropertiesServices`'s JSON-Schema definition
+ * via the `patternProperty` "^[a-zA-Z0-9._-]+$".
+ */
+export interface DefinitionsService {
+  deploy?: DefinitionsDeployment;
+  build?:
+    | string
+    | {
+        context?: string;
+        dockerfile?: string;
+        args?: ListOrDict;
+        ssh?: ListOrDict;
+        labels?: ListOrDict;
+        cache_from?: string[];
+        cache_to?: string[];
+        no_cache?: boolean;
+        network?: string;
+        pull?: boolean;
+        target?: string;
+        shm_size?: number | string;
+        extra_hosts?: ListOrDict;
+        isolation?: string;
+        privileged?: boolean;
+        secrets?: ServiceConfigOrSecret;
+        tags?: string[];
+        platforms?: string[];
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^x-".
+         */
+        [k: string]: unknown;
+      };
+  blkio_config?: {
+    device_read_bps?: BlkioLimit[];
+    device_read_iops?: BlkioLimit[];
+    device_write_bps?: BlkioLimit[];
+    device_write_iops?: BlkioLimit[];
+    weight?: number;
+    weight_device?: BlkioWeight[];
+  };
+  cap_add?: string[];
+  cap_drop?: string[];
+  cgroup?: "host" | "private";
+  cgroup_parent?: string;
+  command?: string | string[];
+  configs?: ServiceConfigOrSecret;
+  container_name?: string;
+  cpu_count?: number;
+  cpu_percent?: number;
+  cpu_shares?: number | string;
+  cpu_quota?: number | string;
+  cpu_period?: number | string;
+  cpu_rt_period?: number | string;
+  cpu_rt_runtime?: number | string;
+  cpus?: number | string;
+  cpuset?: string;
+  credential_spec?: {
+    config?: string;
+    file?: string;
+    registry?: string;
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  depends_on?:
+    | ListOfStrings
+    | {
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^[a-zA-Z0-9._-]+$".
+         */
+        [k: string]: {
+          condition: "service_started" | "service_healthy" | "service_completed_successfully";
+        };
+      };
+  device_cgroup_rules?: ListOfStrings;
+  devices?: string[];
+  dns?: StringOrList;
+  dns_opt?: string[];
+  dns_search?: StringOrList;
+  domainname?: string;
+  entrypoint?: string | string[];
+  env_file?: StringOrList;
+  environment?: ListOrDict;
+  expose?: (string | number)[];
+  extends?:
+    | string
+    | {
+        service: string;
+        file?: string;
+      };
+  external_links?: string[];
+  extra_hosts?: ListOrDict;
+  group_add?: (string | number)[];
+  healthcheck?: DefinitionsHealthcheck;
+  hostname?: string;
+  image?: string;
+  init?: boolean;
+  ipc?: string;
+  isolation?: string;
+  labels?: ListOrDict;
+  links?: string[];
+  logging?: {
+    driver?: string;
+    options?: {
+      /**
+       * This interface was referenced by `undefined`'s JSON-Schema definition
+       * via the `patternProperty` "^.+$".
+       */
+      [k: string]: string | number | null;
+    };
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^x-".
+     */
+    [k: string]: unknown;
+  };
+  mac_address?: string;
+  mem_limit?: number | string;
+  mem_reservation?: string | number;
+  mem_swappiness?: number;
+  memswap_limit?: number | string;
+  network_mode?: string;
+  networks?:
+    | ListOfStrings
+    | {
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^[a-zA-Z0-9._-]+$".
+         */
+        [k: string]: {
+          aliases?: ListOfStrings;
+          ipv4_address?: string;
+          ipv6_address?: string;
+          link_local_ips?: ListOfStrings;
+          priority?: number;
+          /**
+           * This interface was referenced by `undefined`'s JSON-Schema definition
+           * via the `patternProperty` "^x-".
+           */
+          [k: string]: unknown;
+        } | null;
+      };
+  oom_kill_disable?: boolean;
+  oom_score_adj?: number;
+  pid?: string | null;
+  pids_limit?: number | string;
+  platform?: string;
+  ports?: (
+    | number
+    | string
+    | {
+        mode?: string;
+        host_ip?: string;
+        target?: number;
+        published?: string | number;
+        protocol?: string;
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^x-".
+         */
+        [k: string]: unknown;
+      }
+  )[];
+  privileged?: boolean;
+  profiles?: ListOfStrings;
+  pull_policy?: "always" | "never" | "if_not_present" | "build" | "missing";
+  read_only?: boolean;
+  restart?: string;
+  runtime?: string;
+  scale?: number;
+  security_opt?: string[];
+  shm_size?: number | string;
+  secrets?: ServiceConfigOrSecret;
+  sysctls?: ListOrDict;
+  stdin_open?: boolean;
+  stop_grace_period?: string;
+  stop_signal?: string;
+  storage_opt?: {
+    [k: string]: unknown;
+  };
+  tmpfs?: StringOrList;
+  tty?: boolean;
+  ulimits?: {
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^[a-z]+$".
+     */
+    [k: string]:
+      | number
+      | {
+          hard: number;
+          soft: number;
+          /**
+           * This interface was referenced by `undefined`'s JSON-Schema definition
+           * via the `patternProperty` "^x-".
+           */
+          [k: string]: unknown;
+        };
+  };
+  user?: string;
+  uts?: string;
+  userns_mode?: string;
+  volumes?: (
+    | string
+    | {
+        type: string;
+        source?: string;
+        target?: string;
+        read_only?: boolean;
+        consistency?: string;
+        bind?: {
+          propagation?: string;
+          create_host_path?: boolean;
+          selinux?: "z" | "Z";
+          /**
+           * This interface was referenced by `undefined`'s JSON-Schema definition
+           * via the `patternProperty` "^x-".
+           */
+          [k: string]: unknown;
+        };
+        volume?: {
+          nocopy?: boolean;
+          /**
+           * This interface was referenced by `undefined`'s JSON-Schema definition
+           * via the `patternProperty` "^x-".
+           */
+          [k: string]: unknown;
+        };
+        tmpfs?: {
+          size?: number | string;
+          mode?: number;
+          /**
+           * This interface was referenced by `undefined`'s JSON-Schema definition
+           * via the `patternProperty` "^x-".
+           */
+          [k: string]: unknown;
+        };
+        /**
+         * This interface was referenced by `undefined`'s JSON-Schema definition
+         * via the `patternProperty` "^x-".
+         */
+        [k: string]: unknown;
+      }
+  )[];
+  volumes_from?: string[];
+  working_dir?: string;
+  /**
+   * This interface was referenced by `DefinitionsService`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+}
+export interface BlkioLimit {
+  path?: string;
+  rate?: number | string;
+}
+export interface BlkioWeight {
+  path?: string;
+  weight?: number;
+}
+export interface DefinitionsHealthcheck {
+  disable?: boolean;
+  interval?: string;
+  retries?: number;
+  test?: string | string[];
+  timeout?: string;
+  start_period?: string;
+  /**
+   * This interface was referenced by `DefinitionsHealthcheck`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+}
+export interface PropertiesNetworks {
+  [k: string]: DefinitionsNetwork;
+}
+export interface PropertiesVolumes {
+  [k: string]: DefinitionsVolume;
+}
+export interface PropertiesSecrets {
+  [k: string]: DefinitionsSecret;
+}
+/**
+ * This interface was referenced by `PropertiesSecrets`'s JSON-Schema definition
+ * via the `patternProperty` "^[a-zA-Z0-9._-]+$".
+ */
+export interface DefinitionsSecret {
+  name?: string;
+  environment?: string;
+  file?: string;
+  external?:
+    | boolean
+    | {
+        name?: string;
+        [k: string]: unknown;
+      };
+  labels?: ListOrDict;
+  driver?: string;
+  driver_opts?: {
+    /**
+     * This interface was referenced by `undefined`'s JSON-Schema definition
+     * via the `patternProperty` "^.+$".
+     */
+    [k: string]: string | number;
+  };
+  template_driver?: string;
+  /**
+   * This interface was referenced by `DefinitionsSecret`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+}
+export interface PropertiesConfigs {
+  [k: string]: DefinitionsConfig;
+}
+/**
+ * This interface was referenced by `PropertiesConfigs`'s JSON-Schema definition
+ * via the `patternProperty` "^[a-zA-Z0-9._-]+$".
+ */
+export interface DefinitionsConfig {
+  name?: string;
+  file?: string;
+  external?:
+    | boolean
+    | {
+        name?: string;
+        [k: string]: unknown;
+      };
+  labels?: ListOrDict;
+  template_driver?: string;
+  /**
+   * This interface was referenced by `DefinitionsConfig`'s JSON-Schema definition
+   * via the `patternProperty` "^x-".
+   */
+  [k: string]: unknown;
+}

hole-punch-interop/dockerBuildWrapper.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env /bin/bash
+
+CACHING_OPTIONS=""
+# If in CI and we have a defined cache bucket, use caching
+if [[ -n "${CI}" ]] && [[ -n "${AWS_BUCKET}" ]]; then
+    CACHING_OPTIONS="\
+        --cache-to   type=s3,mode=max,bucket=$AWS_BUCKET,region=$AWS_REGION,prefix=buildCache,name=$IMAGE_NAME \
+        --cache-from type=s3,mode=max,bucket=$AWS_BUCKET,region=$AWS_REGION,prefix=buildCache,name=$IMAGE_NAME"
+fi
+
+docker buildx build \
+    --load \
+    -t $IMAGE_NAME $CACHING_OPTIONS "$@"

hole-punch-interop/helpers/cache.ts

@@ -0,0 +1,157 @@
+const AWS_BUCKET = process.env.AWS_BUCKET || 'libp2p-by-tf-aws-bootstrap';
+const scriptDir = __dirname;
+
+import * as crypto from 'crypto';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as child_process from 'child_process';
+import ignore, { Ignore } from 'ignore'
+
+const holePunchInteropDir = path.join(scriptDir, '..')
+const arch = child_process.execSync('docker info -f "{{.Architecture}}"').toString().trim();
+
+enum Mode {
+    LoadCache = 1,
+    PushCache,
+}
+const modeStr = process.argv[2];
+let mode: Mode
+switch (modeStr) {
+    case "push":
+        mode = Mode.PushCache
+        break
+    case "load":
+        mode = Mode.LoadCache
+        break
+    default:
+        throw new Error(`Unknown mode: ${modeStr}`)
+}
+
+(async () => {
+    for (const implFamily of fs.readdirSync(path.join(holePunchInteropDir, 'impl'))) {
+        const ig = ignore()
+
+        addGitignoreIfPresent(ig, path.join(holePunchInteropDir, ".gitignore"))
+        addGitignoreIfPresent(ig, path.join(holePunchInteropDir, "..", ".gitignore"))
+
+        const implFamilyDir = path.join(holePunchInteropDir, 'impl', implFamily)
+
+        addGitignoreIfPresent(ig, path.join(implFamilyDir, ".gitignore"))
+
+        for (const impl of fs.readdirSync(implFamilyDir)) {
+            const implFolder = fs.realpathSync(path.join(implFamilyDir, impl));
+
+            if (!fs.statSync(implFolder).isDirectory()) {
+                continue;
+            }
+
+            await loadCacheOrBuild(implFolder, ig);
+        }
+
+        await loadCacheOrBuild("router", ig);
+        await loadCacheOrBuild("rust-relay", ig);
+    }
+})()
+
+async function loadCacheOrBuild(dir: string, ig: Ignore) {
+    addGitignoreIfPresent(ig, path.join(dir, ".gitignore"))
+
+    // Get all the files in the dir:
+    let files = walkDir(dir)
+    // Turn them into relative paths:
+    files = files.map(f => f.replace(dir + "/", ""))
+    // Ignore files that are in the .gitignore:
+    files = files.filter(ig.createFilter())
+    // Sort them to be deterministic
+    files = files.sort()
+
+    console.log(dir)
+    console.log("Files:", files)
+
+    // Turn them back into absolute paths:
+    files = files.map(f => path.join(dir, f))
+    const cacheKey = await hashFiles(files)
+    console.log("Cache key:", cacheKey)
+
+    if (mode == Mode.PushCache) {
+        console.log("Pushing cache")
+        try {
+            const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, {method: "HEAD"})
+            if (res.ok) {
+                console.log("Cache already exists")
+            } else {
+                // Read image id from image.json
+                const imageID = JSON.parse(fs.readFileSync(path.join(dir, 'image.json')).toString()).imageID;
+                console.log(`Pushing cache for ${dir}: ${imageID}`)
+                child_process.execSync(`docker image save ${imageID} | gzip | aws s3 cp - s3://${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`);
+            }
+        } catch (e) {
+            console.log("Failed to push image cache:", e)
+        }
+    } else if (mode == Mode.LoadCache) {
+        if (fs.existsSync(path.join(dir, 'image.json'))) {
+            console.log("Already built")
+            return;
+        }
+        console.log("Loading cache")
+        let cacheHit = false
+        try {
+            // Check if the cache exists
+            const res = await fetch(`https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz`, {method: "HEAD"})
+            if (res.ok) {
+                const dockerLoadedMsg = child_process.execSync(`curl https://s3.amazonaws.com/${AWS_BUCKET}/imageCache/${cacheKey}-${arch}.tar.gz  | docker image load`).toString();
+                const loadedImageId = dockerLoadedMsg.match(/Loaded image( ID)?: (.*)/)[2];
+                if (loadedImageId) {
+                    console.log(`Cache hit for ${loadedImageId}`);
+                    fs.writeFileSync(path.join(dir, 'image.json'), JSON.stringify({imageID: loadedImageId}) + "\n");
+                    cacheHit = true
+                }
+            } else {
+                console.log("Cache not found")
+            }
+        } catch (e) {
+            console.log("Cache not found:", e)
+        }
+
+        if (cacheHit) {
+            console.log("Building any remaining things from image.json")
+            // We're building using -o image.json. This tells make to
+            // not bother building image.json or anything it depends on.
+            child_process.execSync(`make -o image.json`, {cwd: dir, stdio: 'inherit'})
+        } else {
+            console.log("No cache, building from scratch")
+            child_process.execSync(`make`, {cwd: dir, stdio: "inherit"})
+        }
+    }
+}
+
+function walkDir(dir: string) {
+    let results = [];
+    fs.readdirSync(dir).forEach(f => {
+        let dirPath = path.join(dir, f);
+        let isDirectory = fs.statSync(dirPath).isDirectory();
+        results = isDirectory ? results.concat(walkDir(dirPath)) : results.concat(path.join(dir, f));
+    });
+    return results;
+};
+
+async function hashFiles(files: string[]): Promise<string> {
+    const fileHashes = await Promise.all(
+        files.map(async (file) => {
+            const data = await fs.promises.readFile(file);
+            return crypto.createHash('sha256').update(data).digest('hex');
+        })
+    );
+    return crypto.createHash('sha256').update(fileHashes.join('')).digest('hex');
+}
+
+function addGitignoreIfPresent(ig: Ignore, pathStr: string): boolean {
+    try {
+        if (fs.statSync(pathStr).isFile()) {
+            ig.add(fs.readFileSync(pathStr).toString())
+        }
+        return true
+    } catch {
+        return false
+    }
+}

hole-punch-interop/impl/rust/.gitignore

@@ -0,0 +1,4 @@
+rust-libp2p-*.zip
+rust-libp2p-*
+rust-libp2p-*/*
+image.json

hole-punch-interop/impl/rust/v0.52/Makefile

@@ -0,0 +1,20 @@
+image_name := rust-v0.52
+commitSha := d99fa2f87406cbd1a207b29f8cbcec32b7b180f7
+
+all: image.json
+
+image.json: rust-libp2p-${commitSha}
+	cd rust-libp2p-${commitSha} && IMAGE_NAME=${image_name} ../../../../dockerBuildWrapper.sh -f hole-punching-tests/Dockerfile .
+	docker image inspect ${image_name} -f "{{.Id}}" | \
+		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
+
+rust-libp2p-${commitSha}: rust-libp2p-${commitSha}.zip
+	unzip -o rust-libp2p-${commitSha}.zip
+
+rust-libp2p-${commitSha}.zip:
+	wget -O $@ "https://github.com/libp2p/rust-libp2p/archive/${commitSha}.zip"
+
+clean:
+	rm image.json
+	rm rust-libp2p-*.zip
+	rm -rf rust-libp2p-*

hole-punch-interop/package.json

@@ -0,0 +1,28 @@
+{
+    "name": "libp2p hole-punch test",
+    "version": "0.0.1",
+    "description": "Tests hole-punching across libp2p implementations",
+    "main": "testplans.ts",
+    "scripts": {
+        "test": "ts-node src/*.test.ts && ts-node testplans.ts",
+        "renderResults": "ts-node renderResults.ts",
+        "cache": "ts-node helpers/cache.ts"
+    },
+    "author": "marcopolo",
+    "license": "MIT",
+    "devDependencies": {
+        "ts-node": "^10.9.1",
+        "typescript": "^4.9.3"
+    },
+    "dependencies": {
+        "@types/yargs": "^17.0.19",
+        "csv-parse": "^5.3.3",
+        "csv-stringify": "^6.2.3",
+        "ignore": "^5.2.4",
+        "json-schema-to-typescript": "^11.0.2",
+        "sqlite": "^4.1.2",
+        "sqlite3": "^5.1.2",
+        "yaml": "^2.2.1",
+        "yargs": "^17.6.2"
+    }
+}

hole-punch-interop/renderResults.ts

@@ -0,0 +1,41 @@
+import { generateTable, load, markdownTable } from './src/lib'
+
+// Read results.csv
+export async function render() {
+    const runs = load("results.csv")
+
+    const regex = /(?<implA>.+) x (?<implB>.+) \((?<options>.*)\)/
+    const parsedRuns = runs.map(run => {
+        const match = run.name.match(regex)
+        if (!match || match.groups === undefined) {
+            throw new Error(`Run ID ${run.name} does not match the expected format`);
+        }
+        return {
+            ...run,
+            implA: match.groups.implA,
+            implB: match.groups.implB,
+            options: match.groups.options.split(",").map(option => option.replace("_", " ").trim()),
+        }
+    })
+
+    // Group by options
+    const runsByOptions = parsedRuns.reduce((acc: { [key: string]: any }, run) => {
+        acc[JSON.stringify(run.options)] = [...acc[JSON.stringify(run.options)] || [], run]
+        return acc
+    }, {})
+
+    let outMd = ""
+
+    for (const runGroup of Object.values(runsByOptions)) {
+        outMd += `## Using: ${runGroup[0].options.join(", ")}\n`
+        const table = generateTable(runGroup)
+        outMd += markdownTable(table)
+        outMd += "\n\n"
+    }
+
+    console.log(outMd)
+
+}
+
+render()
+

hole-punch-interop/router/Dockerfile

@@ -0,0 +1,11 @@
+FROM debian:12-slim
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN --mount=type=cache,target=/var/cache/apt apt-get update && apt-get -y install iproute2 nftables jq tcpdump
+
+COPY *.sh /scripts/
+RUN chmod +x /scripts/*.sh
+
+HEALTHCHECK CMD [ "sh", "-c", "test $(cat /tmp/setup_done) = 1" ]
+
+ENTRYPOINT ["./scripts/run.sh"]

hole-punch-interop/router/Makefile

@@ -0,0 +1,10 @@
+image_name := hole-punch-test-router
+
+all: image.json
+
+image.json: Dockerfile run.sh
+	IMAGE_NAME=${image_name} ../dockerBuildWrapper.sh -f Dockerfile .
+	docker image inspect ${image_name} -f "{{.Id}}" | \
+		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
+clean:
+	rm image.json

hole-punch-interop/router/README.md

@@ -0,0 +1,16 @@
+# Router
+
+This directory contains a Debian-based router implemented on top of nftables.
+
+It expects to be run with two network interfaces:
+
+- `eth0`: The "external" interface.
+- `eth1`: The "internal" interface.
+
+The order of these is important.
+The router cannot possibly know which one is which and thus assumes that `eth0` is the external one and `eth1` the internal one.
+The firewall is set up to take incoming traffic on `eth1` and forward + masquerade it to `eth0`.
+
+It also expects an env variable `DELAY_MS` to be set and will apply this delay as part of the routing process[^1].
+
+[^1]: This is done via `tc qdisc` which only works for egress traffic. To ensure the delay applies in both directions, we divide it by 2 and apply it on both interfaces.

hole-punch-interop/router/run.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+set -ex
+
+if [ -z "$DELAY_MS" ]; then
+  echo "Error: DELAY_MS is not set!"
+  exit 1
+fi
+
+ADDR_EXTERNAL=$(ip -json addr show eth0 | jq '.[0].addr_info[0].local' -r)
+SUBNET_INTERNAL=$(ip -json addr show eth1 | jq '.[0].addr_info[0].local + "/" + (.[0].addr_info[0].prefixlen | tostring)' -r)
+
+# Set up NAT
+nft add table ip nat
+nft add chain ip nat postrouting { type nat hook postrouting priority 100 \; }
+nft add rule ip nat postrouting ip saddr $SUBNET_INTERNAL oifname "eth0" snat $ADDR_EXTERNAL
+
+# tc can only apply delays on egress traffic. By setting a delay for both eth0 and eth1, we achieve the active delay passed in as a parameter.
+half_of_delay=$(expr "$DELAY_MS" / 2 )
+param="${half_of_delay}ms"
+
+tc qdisc add dev eth0 root netem delay $param
+tc qdisc add dev eth1 root netem delay $param
+
+echo "1" > /tmp/setup_done # This will be checked by our docker HEALTHCHECK
+
+tail -f /dev/null # Keep it running forever.

hole-punch-interop/rust-relay/.dockerignore

@@ -0,0 +1 @@
+target/

hole-punch-interop/rust-relay/.gitignore

@@ -0,0 +1 @@
+target/

hole-punch-interop/rust-relay/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "relay"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.72"
+env_logger = "0.10.0"
+libp2p = { version = "0.52.1", features = ["tokio", "relay", "ed25519", "quic", "tcp", "yamux", "noise", "macros", "identify", "ping"] }
+log = "0.4.19"
+redis = { version = "0.23.0", default-features = false, features = ["tokio-comp"] }
+tokio = { version = "1.29.1", features = ["rt-multi-thread", "macros"] }

hole-punch-interop/rust-relay/Dockerfile

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1.5-labs
+FROM rust:1.72.0 as builder
+
+RUN rustup target add x86_64-unknown-linux-musl
+RUN --mount=type=cache,target=/var/cache/apt apt-get update && apt-get install -y musl-dev musl-tools
+
+# Run with access to the target cache to speed up builds
+WORKDIR /workspace
+ADD . .
+RUN --mount=type=cache,target=./target \
+    --mount=type=cache,target=/usr/local/cargo/registry \
+    cargo build --release --package relay --target x86_64-unknown-linux-musl
+
+RUN --mount=type=cache,target=./target \
+    mv ./target/x86_64-unknown-linux-musl/release/relay /usr/local/bin/relay
+
+FROM alpine:3
+COPY --from=builder /usr/local/bin/relay /usr/bin/relay
+RUN --mount=type=cache,target=/var/cache/apk apk add iproute2-tc
+
+ENV RUST_BACKTRACE=1
+
+CMD ["/usr/bin/relay"]

hole-punch-interop/rust-relay/Makefile

@@ -0,0 +1,10 @@
+image_name := hole-punch-test-rust-relay
+
+all: image.json
+
+image.json: Cargo.lock src/** Dockerfile
+	IMAGE_NAME=${image_name} ../dockerBuildWrapper.sh -f Dockerfile .
+	docker image inspect ${image_name} -f "{{.Id}}" | \
+		xargs -I {} echo "{\"imageID\": \"{}\"}" > $@
+clean:
+	rm image.json

hole-punch-interop/rust-relay/src/main.rs

@@ -0,0 +1,152 @@
+use anyhow::{bail, Context, Result};
+use libp2p::{
+    core::{
+        multiaddr::{Multiaddr, Protocol},
+        muxing::StreamMuxerBox,
+        transport::Transport,
+        upgrade,
+    },
+    futures::future::Either,
+    futures::StreamExt,
+    identify, identity, noise, ping, quic, relay,
+    swarm::{NetworkBehaviour, SwarmBuilder, SwarmEvent},
+    tcp, yamux, PeerId, Swarm,
+};
+use redis::AsyncCommands;
+use std::net::{IpAddr, Ipv4Addr};
+
+/// The redis key we push the relay's TCP listen address to.
+const RELAY_TCP_ADDRESS: &str = "RELAY_TCP_ADDRESS";
+/// The redis key we push the relay's QUIC listen address to.
+const RELAY_QUIC_ADDRESS: &str = "RELAY_QUIC_ADDRESS";
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    env_logger::builder()
+        .filter_level(log::LevelFilter::Debug)
+        .parse_filters(
+            "netlink_proto=warn,rustls=warn,multistream_select=warn,libp2p_swarm::connection=info",
+        )
+        .parse_default_env()
+        .init();
+
+    let mut swarm = make_swarm()?;
+
+    let tcp_listener_id = swarm.listen_on(tcp_addr(Ipv4Addr::UNSPECIFIED.into()))?;
+    let quic_listener_id = swarm.listen_on(quic_addr(Ipv4Addr::UNSPECIFIED.into()))?;
+
+    loop {
+        match swarm.next().await.expect("Infinite Stream.") {
+            SwarmEvent::NewListenAddr {
+                address,
+                listener_id,
+            } => {
+                let Some(Protocol::Ip4(addr)) = address.iter().next() else {
+                    bail!("Expected first protocol of listen address to be Ip4")
+                };
+
+                if addr.is_loopback() {
+                    log::debug!("Ignoring loop-back address: {address}");
+
+                    continue;
+                }
+
+                swarm.add_external_address(address.clone()); // We know that in our testing network setup, that we are listening on a "publicly-reachable" address.
+
+                log::info!("Listening on {address}");
+
+                let address = address
+                    .with(Protocol::P2p(*swarm.local_peer_id()))
+                    .to_string();
+
+                // Push each address twice because we need to connect two clients.
+
+                let mut redis = RedisClient::new("redis", 6379).await?;
+
+                if listener_id == tcp_listener_id {
+                    redis.push(RELAY_TCP_ADDRESS, &address).await?;
+                    redis.push(RELAY_TCP_ADDRESS, &address).await?;
+                }
+                if listener_id == quic_listener_id {
+                    redis.push(RELAY_QUIC_ADDRESS, &address).await?;
+                    redis.push(RELAY_QUIC_ADDRESS, &address).await?;
+                }
+            }
+            other => {
+                log::trace!("{other:?}")
+            }
+        }
+    }
+}
+
+fn tcp_addr(addr: IpAddr) -> Multiaddr {
+    Multiaddr::empty().with(addr.into()).with(Protocol::Tcp(0))
+}
+
+fn quic_addr(addr: IpAddr) -> Multiaddr {
+    Multiaddr::empty()
+        .with(addr.into())
+        .with(Protocol::Udp(0))
+        .with(Protocol::QuicV1)
+}
+
+fn make_swarm() -> Result<Swarm<Behaviour>> {
+    let local_key = identity::Keypair::generate_ed25519();
+    let local_peer_id = PeerId::from(local_key.public());
+    log::info!("Local peer id: {local_peer_id}");
+
+    let transport = tcp::tokio::Transport::new(tcp::Config::default().nodelay(true))
+        .upgrade(upgrade::Version::V1Lazy)
+        .authenticate(noise::Config::new(&local_key)?)
+        .multiplex(yamux::Config::default())
+        .or_transport(quic::tokio::Transport::new(quic::Config::new(&local_key)))
+        .map(|either_output, _| match either_output {
+            Either::Left((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)),
+            Either::Right((peer_id, muxer)) => (peer_id, StreamMuxerBox::new(muxer)),
+        })
+        .boxed();
+    let behaviour = Behaviour {
+        relay: relay::Behaviour::new(local_peer_id, relay::Config::default()),
+        identify: identify::Behaviour::new(identify::Config::new(
+            "/hole-punch-tests/1".to_owned(),
+            local_key.public(),
+        )),
+        ping: ping::Behaviour::default(),
+    };
+
+    Ok(
+        SwarmBuilder::with_tokio_executor(transport, behaviour, local_peer_id)
+            .substream_upgrade_protocol_override(upgrade::Version::V1Lazy)
+            .build(),
+    )
+}
+
+struct RedisClient {
+    inner: redis::aio::Connection,
+}
+
+impl RedisClient {
+    async fn new(host: &str, port: u16) -> Result<Self> {
+        let client = redis::Client::open(format!("redis://{host}:{port}/"))
+            .context("Bad redis server URL")?;
+        let connection = client
+            .get_async_connection()
+            .await
+            .context("Failed to connect to redis server")?;
+
+        Ok(Self { inner: connection })
+    }
+
+    async fn push(&mut self, key: &str, value: impl ToString) -> Result<()> {
+        self.inner.rpush(key, value.to_string()).await?;
+
+        Ok(())
+    }
+}
+
+#[derive(NetworkBehaviour)]
+struct Behaviour {
+    relay: relay::Behaviour,
+    identify: identify::Behaviour,
+    ping: ping::Behaviour,
+}

hole-punch-interop/src/compose-runner.ts

@@ -0,0 +1,79 @@
+import {promises as fs} from 'fs';
+import path from 'path';
+import {exec as execStd} from 'child_process';
+import util from 'util';
+import {ComposeSpecification} from "../compose-spec/compose-spec";
+import {stringify} from 'yaml';
+import {sanitizeComposeName} from "./lib";
+
+const exec = util.promisify(execStd);
+
+export async function run(compose: ComposeSpecification, rootAssetDir: string, dryRun: boolean): Promise<Report | null> {
+    const sanitizedComposeName = sanitizeComposeName(compose.name)
+    const assetDir = path.join(rootAssetDir, sanitizedComposeName);
+
+    await fs.mkdir(assetDir, { recursive: true })
+
+
+    // Create compose.yaml file
+    // Some docker compose environments don't like the name field to have special characters
+    const composeYmlPath = path.join(assetDir, "docker-compose.yaml");
+    await fs.writeFile(composeYmlPath, stringify({ ...compose, name: sanitizedComposeName }))
+
+    if (dryRun) {
+        return null;
+    }
+
+    const stdoutLogFile = path.join(assetDir, `stdout.log`);
+    const stderrLogFile = path.join(assetDir, `stderr.log`);
+
+    try {
+        const { stdout, stderr } = await exec(`docker compose -f ${composeYmlPath} up --exit-code-from dialer --abort-on-container-exit`, { timeout: 60 * 1000 })
+
+        await fs.writeFile(stdoutLogFile, stdout);
+        await fs.writeFile(stderrLogFile, stderr);
+
+        return JSON.parse(lastStdoutLine(stdout, "dialer", sanitizedComposeName)) as Report
+    } catch (e: unknown) {
+        if (isExecException(e)) {
+            await fs.writeFile(stdoutLogFile, e.stdout)
+            await fs.writeFile(stderrLogFile, e.stderr)
+        }
+
+        throw e
+    } finally {
+        try {
+            await exec(`docker compose -f ${composeYmlPath} down`);
+        } catch (e) {
+            console.log("Failed to compose down", e)
+        }
+    }
+}
+
+export interface ExecException extends Error {
+    cmd?: string | undefined;
+    killed?: boolean | undefined;
+    code?: number | undefined;
+    signal?: NodeJS.Signals | undefined;
+    stdout: string;
+    stderr: string;
+}
+
+function isExecException(candidate: unknown): candidate is ExecException {
+    return candidate && typeof candidate === 'object' && 'cmd' in candidate;
+}
+
+interface Report {
+    rtt_to_holepunched_peer_millis: number
+}
+
+export function lastStdoutLine(stdout: string, component: string, composeName: string): string {
+    const allComponentStdout = stdout.split("\n").filter(line => line.startsWith(`${composeName}-${component}-1`));
+
+    const exitMessage = allComponentStdout.pop();
+    const lastLine = allComponentStdout.pop();
+
+    const [front, componentStdout] = lastLine.split("|");
+
+    return componentStdout.trim()
+}

hole-punch-interop/src/generator.ts

@@ -0,0 +1,172 @@
+import sqlite3 from "sqlite3";
+import {open} from "sqlite";
+import {Version} from "../versions";
+import {ComposeSpecification} from "../compose-spec/compose-spec";
+import {sanitizeComposeName} from "./lib";
+import path from "path";
+
+export async function buildTestSpecs(versions: Array<Version>, nameFilter: string | null, nameIgnore: string | null, routerImageId: string, relayImageId: string, routerDelay: number, relayDelay: number, assetDir: string): Promise<Array<ComposeSpecification>> {
+    sqlite3.verbose();
+
+    const db = await open({
+        // In memory DB. We don't persist this.
+        filename: ":memory:",
+        driver: sqlite3.Database,
+    });
+
+    await db.exec('CREATE TABLE IF NOT EXISTS transports (id string not null, imageID string not null, transport string not null);');
+
+    await Promise.all(
+        versions.flatMap(version => ([
+            db.exec(`INSERT INTO transports (id, imageID, transport) VALUES ${version.transports.map(transport => `("${version.id}", "${version.containerImageID}", "${transport}")`).join(", ")};`)
+        ]))
+    )
+
+    // Generate the testing combinations by SELECT'ing from transports tables the distinct combinations where the transports of the different libp2p implementations match.
+    const queryResults =
+        await db.all(`SELECT DISTINCT a.id as dialer, a.imageID as dialerImage, b.id as listener, b.imageID as listenerImage, a.transport
+                      FROM transports a,
+                           transports b
+                      WHERE a.transport == b.transport;`
+        );
+    await db.close();
+
+    return queryResults
+        .map(testCase => {
+            let name = `${testCase.dialer} x ${testCase.listener} (${testCase.transport})`;
+
+            if (nameFilter && !name.includes(nameFilter)) {
+                return null
+            }
+            if (nameIgnore && name.includes(nameIgnore)) {
+                return null
+            }
+
+            return buildSpec(name, testCase.dialerImage, testCase.listenerImage, routerImageId, relayImageId, testCase.transport, routerDelay, relayDelay, assetDir, {})
+        })
+        .filter(spec => spec !== null)
+}
+
+function buildSpec(name: string, dialerImage: string, listenerImage: string, routerImageId: string, relayImageId: string, transport: string, routerDelay: number, relayDelay: number, assetDir: string, extraEnv: { [key: string]: string }): ComposeSpecification {
+    let internetNetworkName = `${sanitizeComposeName(name)}_internet`
+
+    let startupScriptFn = (actor: "dialer" | "listener") => (`
+        set -ex;
+
+        ROUTER_IP=$$(dig +short ${actor}_router)
+        INTERNET_SUBNET=$$(curl --silent --unix-socket /var/run/docker.sock http://localhost/networks/${internetNetworkName} | jq -r '.IPAM.Config[0].Subnet')
+
+        ip route add $$INTERNET_SUBNET via $$ROUTER_IP dev eth0
+
+        tcpdump -i eth0 -w /tmp/${actor}.pcap &
+
+        sleep 2 # Let tcpdump start up
+
+        hole-punch-client
+    `);
+
+    let relayStartupScript = `
+        set -ex;
+
+        tc qdisc add dev eth0 root netem delay ${relayDelay}ms; # Add a delay to all relayed connections
+
+        /usr/bin/relay
+    `;
+
+    const dockerSocketVolume = "/var/run/docker.sock:/var/run/docker.sock";
+    const tcpDumpVolume = `${path.join(assetDir, sanitizeComposeName(name))}:/tmp:rw`;
+
+    return {
+        name,
+        services: {
+            relay: {
+                depends_on: ["redis"],
+                image: relayImageId,
+                init: true,
+                command: ["/bin/sh", "-c", relayStartupScript],
+                networks: {
+                    internet: {},
+                },
+                cap_add: ["NET_ADMIN"]
+            },
+            dialer_router: {
+                depends_on: ["redis"],
+                image: routerImageId,
+                init: true,
+                environment: {
+                    DELAY_MS: routerDelay
+                },
+                networks: {
+                    lan_dialer: {},
+                    internet: {},
+                },
+                cap_add: ["NET_ADMIN"],
+            },
+            dialer: {
+                depends_on: ["relay", "dialer_router", "redis"],
+                image: dialerImage,
+                init: true,
+                command: ["/bin/sh", "-c", startupScriptFn("dialer")],
+                environment: {
+                    TRANSPORT: transport,
+                    MODE: "dial",
+                },
+                networks: {
+                    lan_dialer: {},
+                },
+                cap_add: ["NET_ADMIN"],
+                volumes: [dockerSocketVolume, tcpDumpVolume]
+            },
+            listener_router: {
+                depends_on: ["redis"],
+                image: routerImageId,
+                init: true,
+                environment: {
+                    DELAY_MS: routerDelay
+                },
+                networks: {
+                    lan_listener: {},
+                    internet: {},
+                },
+                cap_add: ["NET_ADMIN"]
+            },
+            listener: {
+                depends_on: ["relay", "listener_router", "redis"],
+                image: listenerImage,
+                init: true,
+                command: ["/bin/sh", "-c", startupScriptFn("listener")],
+                environment: {
+                    TRANSPORT: transport,
+                    MODE: "listen",
+                },
+                networks: {
+                    lan_listener: {},
+                },
+                cap_add: ["NET_ADMIN"],
+                volumes: [dockerSocketVolume, tcpDumpVolume]
+            },
+            redis: {
+                image: "redis:7-alpine",
+                healthcheck: {
+                    test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
+                },
+                networks: {
+                    internet: {
+                        aliases: ["redis"]
+                    },
+                    lan_dialer: {
+                        aliases: ["redis"]
+                    },
+                    lan_listener: {
+                        aliases: ["redis"]
+                    },
+                }
+            }
+        },
+        networks: {
+            lan_dialer: {},
+            lan_listener: {},
+            internet: {},
+        }
+    }
+}

hole-punch-interop/src/lib.ts

@@ -0,0 +1,124 @@
+import * as csv from "csv-parse/sync";
+import fs from "fs";
+
+export type ResultLine = {
+  name: string;
+  outcome: string;
+  error: string;
+};
+
+export type ParsedResultLine = {
+  name: string;
+  outcome: string;
+  error: string;
+  implA: string;
+  implB: string;
+};
+
+export type ResultFile = ResultLine[];
+
+export type CellRender = (a: string, b: string, line: ResultLine) => string;
+
+/**
+ * called for every cell in the table.
+ *
+ * This is designed to let future implementers add more complex ouput interpretation, with nested tables, etc.
+ */
+export const defaultCellRender: CellRender = (a, b, line) => {
+  let result = ":red_circle:";
+
+  if (line.outcome === "success") {
+    result = ":green_circle:";
+  }
+
+  if (process.env.RUN_URL) {
+    result = `[${result}](${process.env.RUN_URL})`;
+  }
+
+  return result;
+};
+
+export const load = (path: string): ResultFile => {
+  return csv.parse(fs.readFileSync(path, "utf8"), {
+    columns: true,
+    skip_empty_lines: true,
+    delimiter: ",",
+  }) as ResultFile;
+};
+
+export const save = (path: string, content: string) => {
+  fs.writeFileSync(path, content);
+};
+
+type PairOfImplementation = [string, string];
+
+export const listUniqPairs = (pairs: PairOfImplementation[]): string[] => {
+  const uniq = new Set<string>();
+
+  for (const [a, b] of pairs) {
+    uniq.add(a);
+    uniq.add(b);
+  }
+
+  return Array.from(uniq).sort();
+};
+
+export const generateEmptyMatrix = (
+  keys: string[],
+  defaultValue: string
+): string[][] => {
+  const header = [" ", ...keys];
+
+  const matrix = [header];
+  const rowOfDefaultValues = Array<string>(keys.length).fill(defaultValue);
+
+  for (const key of keys) {
+    const row = [key, ...rowOfDefaultValues];
+    matrix.push(row);
+  }
+
+  return matrix;
+};
+
+export const generateTable = (
+  results: Array<ParsedResultLine>,
+  defaultValue: string = ":white_circle:",
+  testedCell: CellRender = defaultCellRender
+): string[][] => {
+  const pairs = results.map(({ implA, implB }) => [implA, implB] as PairOfImplementation);
+  const uniqPairs = listUniqPairs(pairs);
+
+  const matrix = generateEmptyMatrix(uniqPairs, defaultValue);
+  matrix[0][0] = "⬇️ dialer 📞 \\  ➡️ listener 🎧"
+
+  for (const result of results) {
+    const { implA, implB } = result
+    const i = uniqPairs.indexOf(implA);
+    const j = uniqPairs.indexOf(implB);
+
+    const cell = testedCell(implA, implB, result);
+
+    matrix[i + 1][j + 1] = cell;
+  }
+
+  return matrix;
+};
+
+export const markdownTable = (table: string[][]): string => {
+  const wrapped = (x: string) => `| ${x} |`;
+
+  const header = table[0].join(" | ");
+  const separator = table[0].map((x) => "-".repeat(x.length)).join(" | ");
+
+  const rows = table.slice(1).map((row) => row.join(" | "));
+
+  const body = [wrapped(header), wrapped(separator), ...rows.map(wrapped)].join(
+    "\n"
+  );
+
+  return body;
+};
+
+export function sanitizeComposeName(name: string) {
+  return name.replace(/[^a-zA-Z0-9_-]/g, "_");
+}

hole-punch-interop/src/stdoutParser.test.ts

@@ -0,0 +1,21 @@
+import {lastStdoutLine} from "./compose-runner";
+
+let exampleStdout = `
+Attaching to rust-v0_52_x_rust-v0_52__quic_-dialer-1, rust-v0_52_x_rust-v0_52__quic_-dialer_router-1, rust-v0_52_x_rust-v0_52__quic_-listener-1, rust-v0_52_x_rust-v0_52__quic_-listener_router-1, rust-v0_52_x_rust-v0_52__quic_-redis-1, rust-v0_52_x_rust-v0_52__quic_-relay-1
+rust-v0_52_x_rust-v0_52__quic_-redis-1         | 1:C 19 Sep 2023 05:19:20.620 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.
+rust-v0_52_x_rust-v0_52__quic_-redis-1         | 1:C 19 Sep 2023 05:19:20.620 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
+rust-v0_52_x_rust-v0_52__quic_-redis-1         | 1:C 19 Sep 2023 05:19:20.620 * Redis version=7.2.1, bits=64, commit=00000000, modified=0, pid=1, just started
+rust-v0_52_x_rust-v0_52__quic_-redis-1         | 1:C 19 Sep 2023 05:19:20.620 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf
+rust-v0_52_x_rust-v0_52__quic_-redis-1         | 1:M 19 Sep 2023 05:19:20.620 * monotonic clock: POSIX clock_gettime
+rust-v0_52_x_rust-v0_52__quic_-redis-1         | 1:M 19 Sep 2023 05:19:20.621 * Running mode=standalone, port=6379.
+rust-v0_52_x_rust-v0_52__quic_-redis-1         | 1:M 19 Sep 2023 05:19:20.621 * Server initialized
+rust-v0_52_x_rust-v0_52__quic_-redis-1         | 1:M 19 Sep 2023 05:19:20.621 * Ready to accept connections tcp
+rust-v0_52_x_rust-v0_52__quic_-dialer-1         | {"rtt_to_holepunched_peer_millis":201}
+rust-v0_52_x_rust-v0_52__quic_-dialer-1 exited with code 0
+`;
+
+const line = lastStdoutLine(exampleStdout, "dialer", "rust-v0_52_x_rust-v0_52__quic_");
+
+if (line != `{"rtt_to_holepunched_peer_millis":201}`) {
+    throw new Error("Unexpected stdout")
+}

hole-punch-interop/testplans.ts

@@ -0,0 +1,131 @@
+import { buildTestSpecs } from "./src/generator"
+import { Version, versions } from "./versions"
+import { promises as fs } from "fs";
+import {ExecException, run} from "./src/compose-runner"
+import { stringify } from "csv-stringify/sync"
+import { stringify as YAMLStringify } from "yaml"
+import yargs from "yargs/yargs"
+import path from "path";
+
+(async () => {
+    const WorkerCount = parseInt(process.env.WORKER_COUNT || "1")
+    const argv = await yargs(process.argv.slice(2))
+        .options({
+            'name-filter': {
+                description: 'Only run tests including this name',
+                default: "",
+            },
+            'name-ignore': {
+                description: 'Do not run any tests including this name',
+                default: "",
+            },
+            'dry-run': {
+                description: "Don't actually run the test, just generate the compose files",
+                default: false,
+                type: 'boolean'
+            },
+            'extra-versions-dir': {
+                description: 'Look for extra versions in this directory. Version files must be in json format',
+                default: "",
+                type: 'string'
+            },
+            'extra-version': {
+                description: 'Paths to JSON files for additional versions to include in the test matrix',
+                default: [],
+                type: 'array'
+            },
+        })
+        .help()
+        .version(false)
+        .alias('help', 'h').argv;
+    const extraVersionsDir = argv.extraVersionsDir
+    const extraVersions: Array<Version> = []
+    if (extraVersionsDir !== "") {
+        try {
+            const files = await fs.readdir(extraVersionsDir);
+            for (const file of files) {
+                const contents = await fs.readFile(path.join(extraVersionsDir, file))
+                extraVersions.push(...JSON.parse(contents.toString()))
+            }
+        } catch (err) {
+            console.error("Error reading extra versions")
+            console.error(err);
+        }
+    }
+
+    for (let versionPath of argv.extraVersion.filter(p => p !== "")) {
+        const contents = await fs.readFile(versionPath);
+        extraVersions.push(JSON.parse(contents.toString()))
+    }
+
+    let nameFilter: string | null = argv["name-filter"]
+    if (nameFilter === "") {
+        nameFilter = null
+    }
+    let nameIgnore: string | null = argv["name-ignore"]
+    if (nameIgnore === "") {
+        nameIgnore = null
+    }
+
+    let routerImageId = JSON.parse(await fs.readFile(path.join(".", "router", "image.json"), "utf-8")).imageID;
+    let relayImageId = JSON.parse(await fs.readFile(path.join(".", "rust-relay", "image.json"), "utf-8")).imageID;
+
+    const routerDelay = 100;
+    const relayDelay = 25;
+
+    const rttRelayedConnection = routerDelay * 2 + relayDelay * 2;
+    const rttDirectConnection = routerDelay * 2;
+
+    const assetDir = path.join(__dirname, "runs");
+
+    let testSpecs = await buildTestSpecs(versions.concat(extraVersions), nameFilter, nameIgnore, routerImageId, relayImageId, routerDelay, relayDelay, assetDir)
+
+    console.log(`Running ${testSpecs.length} tests`)
+    const failures: Array<{ name: String, e: ExecException }> = []
+    const statuses: Array<string[]> = [["name", "outcome"]]
+    const workers = new Array(WorkerCount).fill({}).map(async () => {
+        while (true) {
+            const testSpec = testSpecs.pop()
+            if (testSpec == null) {
+                return
+            }
+            const name = testSpec.name;
+            if (!name) {
+                console.warn("Skipping testSpec without name")
+                continue;
+            }
+
+            console.log("Running test spec: " + name)
+
+            try {
+                const report = await run(testSpec, assetDir, argv['dry-run'] as boolean);
+
+                if (report != null) {
+                    const rttDifference = Math.abs(report.rtt_to_holepunched_peer_millis - rttDirectConnection);
+
+                    if (rttDifference > 5) {
+                        // Emit a warning but don't do anything for now.
+                        console.warn(`Expected RTT of direct connection to be ~${rttDirectConnection}ms but was ${report.rtt_to_holepunched_peer_millis}ms`)
+                    }
+                }
+
+                statuses.push([name, "success"])
+            } catch (e) {
+                failures.push({ name, e })
+                statuses.push([name, "failure"])
+            }
+        }
+    })
+    await Promise.all(workers)
+
+    console.log(`${failures.length} failures:`)
+
+    for (const [number, {name, e}] of failures.entries()) {
+        console.log(`---------- ${name} ---------- (${number + 1} / ${failures.length})\n`);
+        console.log(e.stderr)
+    }
+
+    await fs.writeFile("results.csv", stringify(statuses))
+
+    console.log("Run complete")
+})()

hole-punch-interop/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "compilerOptions": {
+    "resolveJsonModule": true,
+    "esModuleInterop": true,
+    "target": "ES2015",
+    "moduleResolution": "node"
+  }
+}

hole-punch-interop/versions.ts

@@ -0,0 +1,39 @@
+import fs from "fs"
+
+export type Version = {
+    id: string,
+    // This can be the image ID, or a function that takes the version ID and returns the image ID.
+    // By default it uses the canonicalImageIDLookup.
+    containerImageID?: string,
+    transports: Array<"tcp" | "quic">,
+}
+
+export const versions: Array<Version> = [
+    {
+        id: "rust-v0.52",
+        transports: ["tcp", "quic"],
+    },
+].map((v: Version) => (typeof v.containerImageID === "undefined" ? ({ ...v, containerImageID: canonicalImageIDLookup(v.id) }) : v))
+
+function canonicalImagePath(id: string): string {
+    // Split by implementation and version
+    const [impl, version] = id.split("-v")
+    // Drop the patch version
+    const [major, minor, patch] = version.split(".")
+    let versionFolder = `v${major}.${minor}`
+    if (major === "0" && minor === "0") {
+        // We're still in the 0.0.x phase, so we use the patch version
+        versionFolder = `v0.0.${patch}`
+    }
+    // Read the image ID from the JSON file on the filesystem
+    return `./impl/${impl}/${versionFolder}/image.json`
+}
+
+// Loads the container image id for the given version id. Expects the form of
+// "<impl>-vX.Y.Z" or "<impl>vX.Y" and the image id to be in the file
+// "./impl/<impl>/vX.Y/image.json" or "./impl/<impl>/v0.0.Z/image.json"
+function canonicalImageIDLookup(id: string): string {
+    const imageIDJSON = fs.readFileSync(canonicalImagePath(id), "utf8")
+    const imageID = JSON.parse(imageIDJSON).imageID
+    return imageID
+}