From ca94abae5f6a732ee5efe8de97583e107e1c1af4 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Thu, 9 Apr 2026 19:58:05 +1000
Subject: [PATCH 01/14] Add release tests workflow

Adds a workflow for release tests:
- builds a docker image for launching nodes in the tests (basically has additional nimflags set)
- creates a K8s cluster in Digital Ocean
- one pod in the cluster is dedicated as the test runner (uses the logos-storage-nim-cs-dist-tests:latest image)
- the release will fail if the docker image build or the release tests fail
- the K8s cluster is torn down after the tests finish (failure or not)
---
 .claude/settings.json                         |  28 +++++
 .../codex-dist-tests-do-ams3/backend.tf       |  16 +++
 .../clusters/codex-dist-tests-do-ams3/main.tf |  57 ++++++++++
 .../codex-dist-tests-do-ams3/providers.tf     |   4 +
 .../codex-dist-tests-do-ams3/variables.tf     |  10 ++
 .../codex-dist-tests-do-ams3/versions.tf      |  10 ++
 .../release/clusters/modules/doks/locals.tf   |   4 +
 .github/release/clusters/modules/doks/main.tf |  33 ++++++
 .../release/clusters/modules/doks/outputs.tf  |   5 +
 .../clusters/modules/doks/variables.tf        |  97 +++++++++++++++++
 .../release/clusters/modules/doks/versions.tf |   9 ++
 .github/release/job-release-tests.yaml        |  63 +++++++++++
 .github/workflows/release.yml                 | 100 +++++++++++++++++-
 13 files changed, 434 insertions(+), 2 deletions(-)
 create mode 100644 .claude/settings.json
 create mode 100644 .github/release/clusters/codex-dist-tests-do-ams3/backend.tf
 create mode 100644 .github/release/clusters/codex-dist-tests-do-ams3/main.tf
 create mode 100644 .github/release/clusters/codex-dist-tests-do-ams3/providers.tf
 create mode 100644 .github/release/clusters/codex-dist-tests-do-ams3/variables.tf
 create mode 100644 .github/release/clusters/codex-dist-tests-do-ams3/versions.tf
 create mode 100644 .github/release/clusters/modules/doks/locals.tf
 create mode 100644 .github/release/clusters/modules/doks/main.tf
 create mode 100644 .github/release/clusters/modules/doks/outputs.tf
 create mode 100644 .github/release/clusters/modules/doks/variables.tf
 create mode 100644 .github/release/clusters/modules/doks/versions.tf
 create mode 100644 .github/release/job-release-tests.yaml

diff --git a/.claude/settings.json b/.claude/settings.json
new file mode 100644
index 00000000..9f0feb0f
--- /dev/null
+++ b/.claude/settings.json
@@ -0,0 +1,28 @@
+{
+  "permissions": {
+    "allow": [
+      "Read(//Users/egonat/repos/gmega/logos-storage-docs/**)",
+      "Bash(npm run build)",
+      "Bash(npx astro build)",
+      "Bash(npm install)",
+      "Bash(node_modules/.bin/astro build)",
+      "WebSearch",
+      "Bash(npm test:*)",
+      "Bash(node -e \"import\\(''koffi''\\).then\\(m => console.log\\(Object.keys\\(m.default\\)\\)\\).catch\\(console.error\\)\")",
+      "Bash(timeout 5 node -e \":*)",
+      "Bash(node probe.js)",
+      "Bash(node two-node-test.js)",
+      "WebFetch(domain:github.com)",
+      "WebFetch(domain:api.github.com)",
+      "Read(//Users/egonat/repos/logos-storage/infra-codex/clusters/codex-dist-tests-do-ams3/**)",
+      "Read(//Users/egonat/repos/logos-storage/logos-storage-nim-cs-dist-tests/Tests/CodexReleaseTests/**)",
+      "Read(//Users/egonat/repos/logos-storage/logos-storage-nim-cs-dist-tests/**)",
+      "Bash(grep -E \"\\\\.cs$\")",
+      "Read(//Users/egonat/repos/logos-storage/infra-codex/clusters/modules/doks/**)"
+    ],
+    "additionalDirectories": [
+      "/Users/egonat/repos/gmega/logos-storage-docs/src/content/docs/api",
+      "/Users/egonat/repos/gmega/logos-storage-docs/src/content/docs/tutorials"
+    ]
+  }
+}
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/backend.tf b/.github/release/clusters/codex-dist-tests-do-ams3/backend.tf
new file mode 100644
index 00000000..13512d86
--- /dev/null
+++ b/.github/release/clusters/codex-dist-tests-do-ams3/backend.tf
@@ -0,0 +1,16 @@
+terraform {
+  backend "s3" {
+    endpoints = {
+      s3 = "https://fra1.digitaloceanspaces.com"
+    }
+    bucket = "codex-infra-terraform"
+    key    = "clusters/codex-dist-tests-do-ams3/terraform.tfstate"
+    region = "fra1"
+
+    skip_credentials_validation = true
+    skip_requesting_account_id  = true
+    skip_metadata_api_check     = true
+    skip_region_validation      = true
+    skip_s3_checksum            = true
+  }
+}
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/main.tf b/.github/release/clusters/codex-dist-tests-do-ams3/main.tf
new file mode 100644
index 00000000..265117c2
--- /dev/null
+++ b/.github/release/clusters/codex-dist-tests-do-ams3/main.tf
@@ -0,0 +1,57 @@
+# Kubernetes cluster
+module "doks" {
+  source = "../modules/doks"
+
+  name                            = "codex-dist-tests"
+  region                          = var.region
+  kubernetes_version              = "1.34.5-do.2"
+  kubernetes_ha                   = true
+  kubernetes_auto_upgrade         = false
+  kubernetes_node_pool_name       = "infra-s-4vcpu-16gb-amd"
+  kubernetes_node_pool_size       = "s-4vcpu-16gb-amd"
+  kubernetes_node_pool_auto_scale = true
+  kubernetes_node_pool_min        = 1
+  kubernetes_node_pool_max        = 3
+  kubernetes_node_pool_tags       = ["default", "autoscale"]
+  kubernetes_node_pool_labels = {
+    default-pool  = "true"
+    scaling-type  = "auto"
+    workload-type = "infra"
+  }
+}
+
+# Node pool - Runners CI
+resource "digitalocean_kubernetes_node_pool" "runners-ci" {
+  cluster_id = module.doks.kubernetes_cluster_id
+  name       = "runners-ci-s-2vcpu-8gb-amd"
+  size       = "s-2vcpu-8gb-amd"
+  auto_scale = true
+  min_nodes  = 1
+  max_nodes  = 5
+  tags       = ["runners-ci"]
+
+  labels = {
+    allow-tests-pods = "false"
+    default-pool     = "false"
+    scaling-type     = "auto"
+    workload-type    = "tests-runners-ci"
+  }
+}
+
+# Node pool - Tests Pods
+resource "digitalocean_kubernetes_node_pool" "tests-s-2vcpu-4gb" {
+  cluster_id = module.doks.kubernetes_cluster_id
+  name       = "tests-s-2vcpu-4gb"
+  size       = "s-2vcpu-4gb"
+  auto_scale = true
+  min_nodes  = 1
+  max_nodes  = 10
+  tags       = ["tests-pods"]
+
+  labels = {
+    allow-tests-pods = "true"
+    default-pool     = "false"
+    scaling-type     = "auto"
+    workload-type    = "tests-pods"
+  }
+}
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/providers.tf b/.github/release/clusters/codex-dist-tests-do-ams3/providers.tf
new file mode 100644
index 00000000..1eed7365
--- /dev/null
+++ b/.github/release/clusters/codex-dist-tests-do-ams3/providers.tf
@@ -0,0 +1,4 @@
+# Providers
+provider "digitalocean" {
+  token = var.do_token
+}
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/variables.tf b/.github/release/clusters/codex-dist-tests-do-ams3/variables.tf
new file mode 100644
index 00000000..3eb848b3
--- /dev/null
+++ b/.github/release/clusters/codex-dist-tests-do-ams3/variables.tf
@@ -0,0 +1,10 @@
+variable "region" {
+  description = "DigitalOcean region (e.g. ams3)"
+  type        = string
+}
+
+variable "do_token" {
+  description = "DigitalOcean API token"
+  type        = string
+  sensitive   = true
+}
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/versions.tf b/.github/release/clusters/codex-dist-tests-do-ams3/versions.tf
new file mode 100644
index 00000000..8841865e
--- /dev/null
+++ b/.github/release/clusters/codex-dist-tests-do-ams3/versions.tf
@@ -0,0 +1,10 @@
+# Terraform settings
+terraform {
+  required_version = "~> 1.0"
+  required_providers {
+    digitalocean = {
+      source  = "digitalocean/digitalocean"
+      version = "~> 2.0"
+    }
+  }
+}
diff --git a/.github/release/clusters/modules/doks/locals.tf b/.github/release/clusters/modules/doks/locals.tf
new file mode 100644
index 00000000..f2dd8a14
--- /dev/null
+++ b/.github/release/clusters/modules/doks/locals.tf
@@ -0,0 +1,4 @@
+locals {
+  name           = "${var.name}-do-${var.region}"
+  node_pool_name = "pool-${var.kubernetes_node_pool_size}"
+}
diff --git a/.github/release/clusters/modules/doks/main.tf b/.github/release/clusters/modules/doks/main.tf
new file mode 100644
index 00000000..d945c911
--- /dev/null
+++ b/.github/release/clusters/modules/doks/main.tf
@@ -0,0 +1,33 @@
+# Kubernetes cluster
+resource "digitalocean_kubernetes_cluster" "this" {
+  name         = local.name
+  region       = var.region
+  version      = var.kubernetes_version
+  ha           = var.kubernetes_ha
+  auto_upgrade = var.kubernetes_auto_upgrade
+
+  node_pool {
+    name       = var.kubernetes_node_pool_name
+    size       = var.kubernetes_node_pool_size
+    node_count = var.kubernetes_node_pool_count
+    auto_scale = var.kubernetes_node_pool_auto_scale
+    min_nodes  = var.kubernetes_node_pool_min
+    max_nodes  = var.kubernetes_node_pool_max
+    tags       = var.kubernetes_node_pool_tags
+    labels     = var.kubernetes_node_pool_labels
+
+    dynamic "taint" {
+      for_each = length(var.kubernetes_node_pool_taint) == 0 ? {} : { taint = true }
+      content {
+        key    = lookup(var.kubernetes_node_pool_taint, "key")
+        value  = lookup(var.kubernetes_node_pool_taint, "value")
+        effect = lookup(var.kubernetes_node_pool_taint, "effect")
+      }
+    }
+  }
+
+  maintenance_policy {
+    day        = var.kubernetes_maintenance_day
+    start_time = var.kubernetes_maintenance_start_time
+  }
+}
diff --git a/.github/release/clusters/modules/doks/outputs.tf b/.github/release/clusters/modules/doks/outputs.tf
new file mode 100644
index 00000000..3344ad92
--- /dev/null
+++ b/.github/release/clusters/modules/doks/outputs.tf
@@ -0,0 +1,5 @@
+# Kubernetes cluster
+output "kubernetes_cluster_id" {
+  value       = digitalocean_kubernetes_cluster.this.id
+  description = "A unique ID that can be used to identify and reference a Kubernetes cluster."
+}
diff --git a/.github/release/clusters/modules/doks/variables.tf b/.github/release/clusters/modules/doks/variables.tf
new file mode 100644
index 00000000..e0f3253c
--- /dev/null
+++ b/.github/release/clusters/modules/doks/variables.tf
@@ -0,0 +1,97 @@
+# Main
+variable "name" {
+  type        = string
+  description = "A name for the created resources."
+}
+
+variable "region" {
+  type        = string
+  description = "The DigitalOcean region slug for the resources location."
+}
+
+# Kubernetes Control Plane
+variable "kubernetes_version" {
+  type        = string
+  description = "The slug identifier for the version of Kubernetes used for the cluster."
+}
+
+variable "kubernetes_ha" {
+  type        = bool
+  description = "Enable/disable the high availability control plane for a cluster."
+}
+
+variable "kubernetes_auto_upgrade" {
+  type        = bool
+  description = "A boolean value indicating whether the cluster will be automatically upgraded to new patch releases during its maintenance window."
+}
+
+variable "kubernetes_maintenance_day" {
+  type        = string
+  description = "The day of the maintenance window policy."
+  default     = "sunday"
+}
+
+variable "kubernetes_maintenance_start_time" {
+  type        = string
+  description = "The start time in UTC of the maintenance window policy in 24-hour clock format / HH:MM notation (e.g., 15:00)."
+  default     = "04:00"
+}
+
+# Kubernetes default Node Pool
+variable "kubernetes_node_pool_name" {
+  type        = string
+  description = "A name for the node pool."
+  default     = null
+}
+
+variable "kubernetes_node_pool_size" {
+  type        = string
+  description = "The slug identifier for the type of Droplet to be used as workers in the node pool."
+  default     = null
+}
+
+variable "kubernetes_node_pool_count" {
+  type        = number
+  default     = null
+  description = "The number of Droplet instances in the node pool."
+}
+
+variable "kubernetes_node_pool_auto_scale" {
+  type        = bool
+  description = "Enable auto-scaling of the number of nodes in the node pool within the given min/max range."
+  default     = null
+}
+
+variable "kubernetes_node_pool_min" {
+  type        = number
+  description = "If auto-scaling is enabled, this represents the minimum number of nodes that the node pool can be scaled down to."
+  default     = null
+}
+
+variable "kubernetes_node_pool_max" {
+  type        = number
+  description = "If auto-scaling is enabled, this represents the maximum number of nodes that the node pool can be scaled up to."
+  default     = null
+}
+
+variable "kubernetes_node_pool_tags" {
+  type        = list(any)
+  description = "A list of tag names applied to the node pool."
+  default     = ["default", "autoscale"]
+}
+
+variable "kubernetes_node_pool_labels" {
+  type        = map(string)
+  description = "A map of key/value pairs to apply to nodes in the pool."
+  default = {
+    default-pool = "true"
+    scaling-type = "auto"
+  }
+}
+
+variable "kubernetes_node_pool_taint" {
+  type        = map(string)
+  description = "A block representing a taint applied to all nodes in the pool."
+  default = {
+  }
+}
diff --git a/.github/release/clusters/modules/doks/versions.tf b/.github/release/clusters/modules/doks/versions.tf
new file mode 100644
index 00000000..21efa07e
--- /dev/null
+++ b/.github/release/clusters/modules/doks/versions.tf
@@ -0,0 +1,9 @@
+# Terraform settings
+terraform {
+  required_providers {
+    digitalocean = {
+      source  = "digitalocean/digitalocean"
+      version = "~> 2.0"
+    }
+  }
+}
diff --git a/.github/release/job-release-tests.yaml b/.github/release/job-release-tests.yaml
new file mode 100644
index 00000000..4c1b636e
--- /dev/null
+++ b/.github/release/job-release-tests.yaml
@@ -0,0 +1,63 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${NAMEPREFIX}
+  namespace: ${NAMESPACE}
+  labels:
+    name: ${NAMEPREFIX}
+    runid: ${RUNID}
+spec:
+  ttlSecondsAfterFinished: 86400
+  backoffLimit: 0
+  template:
+    metadata:
+      name: ${NAMEPREFIX}
+      labels:
+        app: ${TEST_TYPE}-runner
+        name: ${NAMEPREFIX}
+        runid: ${RUNID}
+    spec:
+      priorityClassName: system-node-critical
+      nodeSelector:
+        workload-type: "tests-runners-ci"
+      containers:
+      - name: runner
+        image: logosstorage/cs-codex-dist-tests:latest
+        imagePullPolicy: Always
+        resources:
+          requests:
+            memory: "1Gi"
+        env:
+        - name: KUBECONFIG
+          value: "/opt/kubeconfig.yaml"
+        - name: LOGPATH
+          value: "/var/log/codex-${TEST_TYPE}"
+        - name: NAMESPACE
+          value: "${NAMESPACE}"
+        - name: BRANCH
+          value: "${BRANCH}"
+        - name: SOURCE
+          value: "${SOURCE}"
+        - name: RUNID
+          value: "${RUNID}"
+        - name: CODEXDOCKERIMAGE
+          value: "${CODEXDOCKERIMAGE}"
+        - name: TESTID
+          value: "${TESTID}"
+        - name: TESTS_TYPE
+          value: "${TEST_TYPE}"
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /opt/kubeconfig.yaml
+          subPath: kubeconfig.yaml
+        - name: logs
+          mountPath: /var/log/codex-${TEST_TYPE}
+        args: ${COMMAND}
+      restartPolicy: Never
+      volumes:
+        - name: kubeconfig
+          secret:
+            secretName: codex-dist-tests-app-kubeconfig
+        - name: logs
+          hostPath:
+            path: /var/log/codex-${TEST_TYPE}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0b64e231..65ea36d5 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -179,11 +179,107 @@ jobs:
           path: ${{ env.build_dir }}/${{ env.c_bindings_lib }}.zip
           if-no-files-found: error
 
+  # Build Docker dist-tests image
+  build-docker-dist-tests:
+    name: Build Docker dist-tests image
+    if: github.ref_type == 'tag'
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      nimflags: '-d:disableMarchNative -d:storage_enable_api_debug_peers=true -d:storage_enable_log_counter=true'
+      nat_ip_auto: true
+      tag_latest: false
+      tag_stable: false
+      tag_suffix: dist-tests
+      tag_sha: false
+    secrets: inherit
+
+  # Release tests
+  release-tests:
+    name: Release Tests
+    runs-on: ubuntu-latest
+    if: github.ref_type == 'tag'
+    needs: build-docker-dist-tests
+    env:
+      TF_VAR_region: ams3
+      TF_VAR_do_token: ${{ secrets.DO_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_KEY }}
+      CODEXDOCKERIMAGE: ${{ needs.build-docker-dist-tests.outputs.logos_storage_image }}
+      NAMESPACE: default
+      TEST_TYPE: release-tests
+      BRANCH: master
+      SOURCE: https://github.com/logos-storage/logos-storage-nim-cs-dist-tests
+      TF_DIR: .github/release/clusters/codex-dist-tests-do-ams3
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+
+      - name: Setup doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DO_TOKEN }}
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v4
+        with:
+          version: v1.30.5
+
+      - name: Terraform init
+        working-directory: ${{ env.TF_DIR }}
+        run: terraform init
+
+      - name: Terraform apply
+        working-directory: ${{ env.TF_DIR }}
+        run: terraform apply -auto-approve
+
+      - name: Get kubeconfig
+        run: doctl kubernetes cluster kubeconfig save codex-dist-tests-do-ams3
+
+      - name: Create in-cluster app kubeconfig secret
+        run: |
+          kubectl create secret generic codex-dist-tests-app-kubeconfig \
+            --from-file=kubeconfig.yaml=$HOME/.kube/config \
+            -n ${{ env.NAMESPACE }}
+
+      - name: Set run variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "NAMEPREFIX=r-tests-${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"dotnet test Tests/CodexReleaseTests"')" >> $GITHUB_ENV
+
+      - name: Deploy test job
+        run: envsubst < .github/release/job-release-tests.yaml | kubectl apply -f -
+
+      - name: Stream test logs
+        run: |
+          kubectl logs -l job-name=${{ env.NAMEPREFIX }} \
+            -n ${{ env.NAMESPACE }} \
+            --follow \
+            --pod-running-timeout=300s
+
+      - name: Check job status
+        run: |
+          sleep 5
+          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} \
+            -o jsonpath='{.status.conditions[0].type}')
+          echo "Job status: $job_status"
+          [[ "$job_status" == "Complete" ]] || exit 1
+
+      - name: Terraform destroy
+        if: always()
+        working-directory: ${{ env.TF_DIR }}
+        run: terraform destroy -auto-approve
+
   # Release
   release:
     runs-on: ubuntu-latest
-    needs: build
-    if: success() || failure()
+    needs: [build, release-tests]
+    if: needs.build.result == 'success' && needs.release-tests.result == 'success'
     steps:
       - name: Set conditional env variables
         shell: bash

From ba28e01af5a42f00f76b6a27a86024f8c92a4026 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Thu, 9 Apr 2026 20:33:16 +1000
Subject: [PATCH 02/14] Add workflow input for cs-dist-tests docker image

By default, the logosstorage/logos-storage-nim-cs-dist-tests:latest image will be used for the test runner in the release tests. However, if developers want to run the release tests and test changes to the runner (eg changes in the logos-storage-nim-cs-dist-tests repo), they can push their changes to a branch and manually run the `docker-runner` workflow in the logos-storage-nim-cs-dist-tests repo. This will create an image like logosstorage/logos-storage-nim-cs-dist-tests:sha-c0465a5. This image can then be used as a release tests workflow input for 'cs-dist-tests runner image'
---
 .github/release/job-release-tests.yaml | 2 +-
 .github/workflows/release.yml          | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/release/job-release-tests.yaml b/.github/release/job-release-tests.yaml
index 4c1b636e..710c89a8 100644
--- a/.github/release/job-release-tests.yaml
+++ b/.github/release/job-release-tests.yaml
@@ -22,7 +22,7 @@ spec:
         workload-type: "tests-runners-ci"
       containers:
       - name: runner
-        image: logosstorage/cs-codex-dist-tests:latest
+        image: ${RUNNER_IMAGE}
         imagePullPolicy: Always
         resources:
           requests:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 65ea36d5..d8117533 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -7,6 +7,12 @@ on:
     branches:
       - master
   workflow_dispatch:
+    inputs:
+      runner_image:
+        description: 'cs-dist-tests runner image (default: logosstorage/cs-codex-dist-tests:latest)'
+        required: false
+        default: 'logosstorage/cs-codex-dist-tests:latest'
+        type: string
 
 env:
   cache_nonce: 0 # Allows for easily busting actions/cache caches
@@ -205,6 +211,7 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_KEY }}
       CODEXDOCKERIMAGE: ${{ needs.build-docker-dist-tests.outputs.logos_storage_image }}
+      RUNNER_IMAGE: ${{ inputs.runner_image || 'logosstorage/cs-codex-dist-tests:latest' }}
       NAMESPACE: default
       TEST_TYPE: release-tests
       BRANCH: master

From f9f5305b4bd98a2783820f66915f98f47a041b41 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Thu, 9 Apr 2026 21:35:15 +1000
Subject: [PATCH 03/14] update workflow inputs, k8s namespace

- remove RUNNER_IMAGE because the cs-dist-tests image is dumb -- it clones the cs-dist-tests repo, checkouts the branch in BRANCH and then runs the release tests. So instead, always use the :latest image (which is built when there are commits to master)
- add the BRANCH workflow input so you can test cs-dist-test changes in the runner if needed
- remove COMMAND arg, it's always going to be 'dotnet test Tests/CodexReleaseTests'
- remove NAMESPACE env variable and just use 'default'. The cluster is ephermal and so all resources deployed are for the release tests, no namespaces needed.
---
 .github/release/job-release-tests.yaml |  8 +++-----
 .github/workflows/release.yml          | 19 ++++++++-----------
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/.github/release/job-release-tests.yaml b/.github/release/job-release-tests.yaml
index 710c89a8..043055f2 100644
--- a/.github/release/job-release-tests.yaml
+++ b/.github/release/job-release-tests.yaml
@@ -2,7 +2,7 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: ${NAMEPREFIX}
-  namespace: ${NAMESPACE}
+  namespace: default
   labels:
     name: ${NAMEPREFIX}
     runid: ${RUNID}
@@ -22,7 +22,7 @@ spec:
         workload-type: "tests-runners-ci"
       containers:
       - name: runner
-        image: ${RUNNER_IMAGE}
+        image: logosstorage/cs-codex-dist-tests:latest
         imagePullPolicy: Always
         resources:
           requests:
@@ -32,8 +32,6 @@ spec:
           value: "/opt/kubeconfig.yaml"
         - name: LOGPATH
           value: "/var/log/codex-${TEST_TYPE}"
-        - name: NAMESPACE
-          value: "${NAMESPACE}"
         - name: BRANCH
           value: "${BRANCH}"
         - name: SOURCE
@@ -52,7 +50,7 @@ spec:
           subPath: kubeconfig.yaml
         - name: logs
           mountPath: /var/log/codex-${TEST_TYPE}
-        args: ${COMMAND}
+        args: ["dotnet", "test", "Tests/CodexReleaseTests"]
       restartPolicy: Never
       volumes:
         - name: kubeconfig
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index d8117533..0946b5e0 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -8,10 +8,10 @@ on:
       - master
   workflow_dispatch:
     inputs:
-      runner_image:
-        description: 'cs-dist-tests runner image (default: logosstorage/cs-codex-dist-tests:latest)'
+      branch:
+        description: 'cs-dist-tests branch to run tests from'
         required: false
-        default: 'logosstorage/cs-codex-dist-tests:latest'
+        default: 'master'
         type: string
 
 env:
@@ -211,10 +211,8 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_KEY }}
       CODEXDOCKERIMAGE: ${{ needs.build-docker-dist-tests.outputs.logos_storage_image }}
-      RUNNER_IMAGE: ${{ inputs.runner_image || 'logosstorage/cs-codex-dist-tests:latest' }}
-      NAMESPACE: default
       TEST_TYPE: release-tests
-      BRANCH: master
+      BRANCH: ${{ inputs.branch || 'master' }}
       SOURCE: https://github.com/logos-storage/logos-storage-nim-cs-dist-tests
       TF_DIR: .github/release/clusters/codex-dist-tests-do-ams3
     steps:
@@ -249,7 +247,7 @@ jobs:
         run: |
           kubectl create secret generic codex-dist-tests-app-kubeconfig \
             --from-file=kubeconfig.yaml=$HOME/.kube/config \
-            -n ${{ env.NAMESPACE }}
+            -n default
 
       - name: Set run variables
         run: |
@@ -257,22 +255,21 @@ jobs:
           echo "RUNID=${RUNID}" >> $GITHUB_ENV
           echo "NAMEPREFIX=r-tests-${RUNID}" >> $GITHUB_ENV
           echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-          echo "COMMAND=$(jq -c 'split(" ")' <<< '"dotnet test Tests/CodexReleaseTests"')" >> $GITHUB_ENV
 
       - name: Deploy test job
         run: envsubst < .github/release/job-release-tests.yaml | kubectl apply -f -
 
       - name: Stream test logs
         run: |
-          kubectl logs -l job-name=${{ env.NAMEPREFIX }} \
-            -n ${{ env.NAMESPACE }} \
+          kubectl logs -l job-name=$NAMEPREFIX \
+            -n default \
             --follow \
             --pod-running-timeout=300s
 
       - name: Check job status
         run: |
           sleep 5
-          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} \
+          job_status=$(kubectl get jobs $NAMEPREFIX -n default \
             -o jsonpath='{.status.conditions[0].type}')
           echo "Job status: $job_status"
           [[ "$job_status" == "Complete" ]] || exit 1

From 516f277b2f3025953c7e24a5548f896257d17d75 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Fri, 10 Apr 2026 17:37:23 +1000
Subject: [PATCH 04/14] Allow workflows to be tested manually by branch

---
 .github/workflows/release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0946b5e0..26c5c024 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -188,7 +188,7 @@ jobs:
   # Build Docker dist-tests image
   build-docker-dist-tests:
     name: Build Docker dist-tests image
-    if: github.ref_type == 'tag'
+    if: github.ref_type == 'tag' || github.event_name == 'workflow_dispatch'
     uses: ./.github/workflows/docker-reusable.yml
     with:
       nimflags: '-d:disableMarchNative -d:storage_enable_api_debug_peers=true -d:storage_enable_log_counter=true'
@@ -203,7 +203,7 @@ jobs:
   release-tests:
     name: Release Tests
     runs-on: ubuntu-latest
-    if: github.ref_type == 'tag'
+    if: github.ref_type == 'tag' || github.event_name == 'workflow_dispatch'
     needs: build-docker-dist-tests
     env:
       TF_VAR_region: ams3

From fca52d2cc5bfc71a1fbf865bc90db0faa1ecb6be Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Fri, 10 Apr 2026 19:07:35 +1000
Subject: [PATCH 05/14] prefix repository secret names with RELEASE_TESTS_

---
 .github/workflows/release.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 26c5c024..f34caa84 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -207,9 +207,9 @@ jobs:
     needs: build-docker-dist-tests
     env:
       TF_VAR_region: ams3
-      TF_VAR_do_token: ${{ secrets.DO_TOKEN }}
-      AWS_ACCESS_KEY_ID: ${{ secrets.SPACES_ACCESS_KEY }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.SPACES_SECRET_KEY }}
+      TF_VAR_do_token: ${{ secrets.RELEASE_TESTS_DO_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.RELEASE_TESTS_SPACES_ACCESS_KEY }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.RELEASE_TESTS_SPACES_SECRET_KEY }}
       CODEXDOCKERIMAGE: ${{ needs.build-docker-dist-tests.outputs.logos_storage_image }}
       TEST_TYPE: release-tests
       BRANCH: ${{ inputs.branch || 'master' }}
@@ -225,7 +225,7 @@ jobs:
       - name: Setup doctl
         uses: digitalocean/action-doctl@v2
         with:
-          token: ${{ secrets.DO_TOKEN }}
+          token: ${{ secrets.RELEASE_TESTS_DO_TOKEN }}
 
       - name: Setup kubectl
         uses: azure/setup-kubectl@v4

From d84b4eb48e773b969b16ada1310fe547f82a3049 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Fri, 10 Apr 2026 20:31:12 +1000
Subject: [PATCH 06/14] wait for pod to start before streaming logs

---
 .github/workflows/release.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f34caa84..f186de2d 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -259,12 +259,19 @@ jobs:
       - name: Deploy test job
         run: envsubst < .github/release/job-release-tests.yaml | kubectl apply -f -
 
+      - name: Wait for test pod to start
+        run: |
+          kubectl wait pod \
+            -l job-name=$NAMEPREFIX \
+            -n default \
+            --for=condition=Ready \
+            --timeout=300s
+
       - name: Stream test logs
         run: |
           kubectl logs -l job-name=$NAMEPREFIX \
             -n default \
-            --follow \
-            --pod-running-timeout=300s
+            --follow
 
       - name: Check job status
         run: |

From 9157005cdfc74ef90fe85af851ace3922ad7bd77 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Fri, 10 Apr 2026 21:40:13 +1000
Subject: [PATCH 07/14] WIP update cs-dist-tests docker image tag

The change to the cs-dist-tests image name was to test if installing doctl to the image would fix the release tests not being able to authenticate into the cluster. This is mainly due to the kubeconfig being generated and stored in a DO secret, as opposed to a static kubeconfig for a permanent cluster as before.

IMPORTANT:
The image tag should be changed back to 'latest'!
---
 .github/release/job-release-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/release/job-release-tests.yaml b/.github/release/job-release-tests.yaml
index 043055f2..4e0c9c01 100644
--- a/.github/release/job-release-tests.yaml
+++ b/.github/release/job-release-tests.yaml
@@ -22,7 +22,7 @@ spec:
         workload-type: "tests-runners-ci"
       containers:
       - name: runner
-        image: logosstorage/cs-codex-dist-tests:latest
+        image: logosstorage/cs-codex-dist-tests:sha-507fdb6
         imagePullPolicy: Always
         resources:
           requests:

From 2c1c625c907201254a0626df9fc95f49333f4589 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Fri, 10 Apr 2026 22:29:30 +1000
Subject: [PATCH 08/14] restore using latest cs-dist-tests image

---
 .github/release/job-release-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/release/job-release-tests.yaml b/.github/release/job-release-tests.yaml
index 4e0c9c01..043055f2 100644
--- a/.github/release/job-release-tests.yaml
+++ b/.github/release/job-release-tests.yaml
@@ -22,7 +22,7 @@ spec:
         workload-type: "tests-runners-ci"
       containers:
       - name: runner
-        image: logosstorage/cs-codex-dist-tests:sha-507fdb6
+        image: logosstorage/cs-codex-dist-tests:latest
         imagePullPolicy: Always
         resources:
           requests:

From 42239db2ea92d94c4d453941fe9f3b1f1fee3bf1 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Fri, 10 Apr 2026 22:32:15 +1000
Subject: [PATCH 09/14] Create static kubeconfig with bearer token

Replace the use of doctl as a credential manager for executing k8s calls with a freshly created bearer token (expires after 2h). Avoids passing a DO personal access token to the cs-dist-tests runner pod.
---
 .github/release/kubeconfig-template.yaml | 17 +++++++++++++++++
 .github/workflows/release.yml            | 11 ++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 .github/release/kubeconfig-template.yaml

diff --git a/.github/release/kubeconfig-template.yaml b/.github/release/kubeconfig-template.yaml
new file mode 100644
index 00000000..5dd36976
--- /dev/null
+++ b/.github/release/kubeconfig-template.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority-data: ${CA}
+    server: ${SERVER}
+  name: release-tests
+contexts:
+- context:
+    cluster: release-tests
+    user: release-tests-runner
+  name: release-tests
+current-context: release-tests
+users:
+- name: release-tests-runner
+  user:
+    token: ${TOKEN}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f186de2d..2f4731a7 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -245,8 +245,17 @@ jobs:
 
       - name: Create in-cluster app kubeconfig secret
         run: |
+          kubectl create serviceaccount release-tests-runner -n default
+          kubectl create clusterrolebinding release-tests-runner \
+            --clusterrole=cluster-admin \
+            --serviceaccount=default:release-tests-runner
+
+          TOKEN=$(kubectl create token release-tests-runner -n default --duration=2h)
+          SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
+          CA=$(kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}')
+
           kubectl create secret generic codex-dist-tests-app-kubeconfig \
-            --from-file=kubeconfig.yaml=$HOME/.kube/config \
+            --from-file=kubeconfig.yaml=<(envsubst < .github/release/kubeconfig-template.yaml) \
             -n default
 
       - name: Set run variables

From 05ece7e419eef67e478e3ba0606473953359503d Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Fri, 10 Apr 2026 23:06:42 +1000
Subject: [PATCH 10/14] export kubeconfig values so template works

---
 .github/workflows/release.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2f4731a7..42b759db 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -250,9 +250,9 @@ jobs:
             --clusterrole=cluster-admin \
             --serviceaccount=default:release-tests-runner
 
-          TOKEN=$(kubectl create token release-tests-runner -n default --duration=2h)
-          SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
-          CA=$(kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}')
+          export TOKEN=$(kubectl create token release-tests-runner -n default --duration=2h)
+          export SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
+          export CA=$(kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}')
 
           kubectl create secret generic codex-dist-tests-app-kubeconfig \
             --from-file=kubeconfig.yaml=<(envsubst < .github/release/kubeconfig-template.yaml) \

From 746f30d85942d5df2ce72c3b7e1c5628d0475ac6 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Mon, 13 Apr 2026 13:01:17 +1000
Subject: [PATCH 11/14] Update workflow success condition

---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 42b759db..9cdc4d4c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -288,7 +288,7 @@ jobs:
           job_status=$(kubectl get jobs $NAMEPREFIX -n default \
             -o jsonpath='{.status.conditions[0].type}')
           echo "Job status: $job_status"
-          [[ "$job_status" == "Complete" ]] || exit 1
+          [[ "$job_status" == "SuccessCriteriaMet" ]] || exit 1
 
       - name: Terraform destroy
         if: always()

From 65f101891d476680adce57770043703f4627da0a Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Thu, 16 Apr 2026 18:06:58 +1000
Subject: [PATCH 12/14] chore: rename Codex references to Logos Storage in
 release tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replace all "Codex" branding in the release test workflow and supporting
files: rename the K8s cluster, Terraform state key, secret, log paths,
env var (CODEXDOCKERIMAGE → STORAGEDOCKERIMAGE), and test runner image
(cs-codex-dist-tests → logos-storage-dist-tests) to align with the
already-updated logos-storage-nim-cs-dist-tests repo in https://github.com/logos-storage/logos-storage-nim-cs-dist-tests/pull/124. Also fix the
dotnet test path to the correct Tests/LogosStorageReleaseTests directory.
---
 .../backend.tf                                   |  2 +-
 .../main.tf                                      |  2 +-
 .../providers.tf                                 |  0
 .../variables.tf                                 |  0
 .../versions.tf                                  |  0
 .github/release/job-release-tests.yaml           | 16 ++++++++--------
 .github/workflows/release.yml                    | 12 ++++++------
 7 files changed, 16 insertions(+), 16 deletions(-)
 rename .github/release/clusters/{codex-dist-tests-do-ams3 => logos-storage-dist-tests-do-ams3}/backend.tf (82%)
 rename .github/release/clusters/{codex-dist-tests-do-ams3 => logos-storage-dist-tests-do-ams3}/main.tf (96%)
 rename .github/release/clusters/{codex-dist-tests-do-ams3 => logos-storage-dist-tests-do-ams3}/providers.tf (100%)
 rename .github/release/clusters/{codex-dist-tests-do-ams3 => logos-storage-dist-tests-do-ams3}/variables.tf (100%)
 rename .github/release/clusters/{codex-dist-tests-do-ams3 => logos-storage-dist-tests-do-ams3}/versions.tf (100%)

diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/backend.tf b/.github/release/clusters/logos-storage-dist-tests-do-ams3/backend.tf
similarity index 82%
rename from .github/release/clusters/codex-dist-tests-do-ams3/backend.tf
rename to .github/release/clusters/logos-storage-dist-tests-do-ams3/backend.tf
index 13512d86..e6b8c6b4 100644
--- a/.github/release/clusters/codex-dist-tests-do-ams3/backend.tf
+++ b/.github/release/clusters/logos-storage-dist-tests-do-ams3/backend.tf
@@ -4,7 +4,7 @@ terraform {
       s3 = "https://fra1.digitaloceanspaces.com"
     }
     bucket = "codex-infra-terraform"
-    key    = "clusters/codex-dist-tests-do-ams3/terraform.tfstate"
+    key    = "clusters/logos-storage-dist-tests-do-ams3/terraform.tfstate"
     region = "fra1"
 
     skip_credentials_validation = true
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/main.tf b/.github/release/clusters/logos-storage-dist-tests-do-ams3/main.tf
similarity index 96%
rename from .github/release/clusters/codex-dist-tests-do-ams3/main.tf
rename to .github/release/clusters/logos-storage-dist-tests-do-ams3/main.tf
index 265117c2..3facf793 100644
--- a/.github/release/clusters/codex-dist-tests-do-ams3/main.tf
+++ b/.github/release/clusters/logos-storage-dist-tests-do-ams3/main.tf
@@ -2,7 +2,7 @@
 module "doks" {
   source = "../modules/doks"
 
-  name                            = "codex-dist-tests"
+  name                            = "logos-storage-dist-tests"
   region                          = var.region
   kubernetes_version              = "1.34.5-do.2"
   kubernetes_ha                   = true
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/providers.tf b/.github/release/clusters/logos-storage-dist-tests-do-ams3/providers.tf
similarity index 100%
rename from .github/release/clusters/codex-dist-tests-do-ams3/providers.tf
rename to .github/release/clusters/logos-storage-dist-tests-do-ams3/providers.tf
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/variables.tf b/.github/release/clusters/logos-storage-dist-tests-do-ams3/variables.tf
similarity index 100%
rename from .github/release/clusters/codex-dist-tests-do-ams3/variables.tf
rename to .github/release/clusters/logos-storage-dist-tests-do-ams3/variables.tf
diff --git a/.github/release/clusters/codex-dist-tests-do-ams3/versions.tf b/.github/release/clusters/logos-storage-dist-tests-do-ams3/versions.tf
similarity index 100%
rename from .github/release/clusters/codex-dist-tests-do-ams3/versions.tf
rename to .github/release/clusters/logos-storage-dist-tests-do-ams3/versions.tf
diff --git a/.github/release/job-release-tests.yaml b/.github/release/job-release-tests.yaml
index 043055f2..3d56fe01 100644
--- a/.github/release/job-release-tests.yaml
+++ b/.github/release/job-release-tests.yaml
@@ -22,7 +22,7 @@ spec:
         workload-type: "tests-runners-ci"
       containers:
       - name: runner
-        image: logosstorage/cs-codex-dist-tests:latest
+        image: logosstorage/logos-storage-dist-tests:latest
         imagePullPolicy: Always
         resources:
           requests:
@@ -31,15 +31,15 @@ spec:
         - name: KUBECONFIG
           value: "/opt/kubeconfig.yaml"
         - name: LOGPATH
-          value: "/var/log/codex-${TEST_TYPE}"
+          value: "/var/log/storage-${TEST_TYPE}"
         - name: BRANCH
           value: "${BRANCH}"
         - name: SOURCE
           value: "${SOURCE}"
         - name: RUNID
           value: "${RUNID}"
-        - name: CODEXDOCKERIMAGE
-          value: "${CODEXDOCKERIMAGE}"
+        - name: STORAGEDOCKERIMAGE
+          value: "${STORAGEDOCKERIMAGE}"
         - name: TESTID
           value: "${TESTID}"
         - name: TESTS_TYPE
@@ -49,13 +49,13 @@ spec:
           mountPath: /opt/kubeconfig.yaml
           subPath: kubeconfig.yaml
         - name: logs
-          mountPath: /var/log/codex-${TEST_TYPE}
-        args: ["dotnet", "test", "Tests/CodexReleaseTests"]
+          mountPath: /var/log/storage-${TEST_TYPE}
+        args: ["dotnet", "test", "Tests/LogosStorageReleaseTests"]
       restartPolicy: Never
       volumes:
         - name: kubeconfig
           secret:
-            secretName: codex-dist-tests-app-kubeconfig
+            secretName: storage-dist-tests-app-kubeconfig
         - name: logs
           hostPath:
-            path: /var/log/codex-${TEST_TYPE}
+            path: /var/log/storage-${TEST_TYPE}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 9cdc4d4c..61aa7933 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -9,7 +9,7 @@ on:
   workflow_dispatch:
     inputs:
       branch:
-        description: 'cs-dist-tests branch to run tests from'
+        description: 'dist-tests branch to run tests from'
         required: false
         default: 'master'
         type: string
@@ -185,7 +185,7 @@ jobs:
           path: ${{ env.build_dir }}/${{ env.c_bindings_lib }}.zip
           if-no-files-found: error
 
-  # Build Docker dist-tests image
+  # Build Docker logosstorage/logos-storage-nim:latest-dist-tests image for Logos Storage nodes in the cluster
   build-docker-dist-tests:
     name: Build Docker dist-tests image
     if: github.ref_type == 'tag' || github.event_name == 'workflow_dispatch'
@@ -210,11 +210,11 @@ jobs:
       TF_VAR_do_token: ${{ secrets.RELEASE_TESTS_DO_TOKEN }}
       AWS_ACCESS_KEY_ID: ${{ secrets.RELEASE_TESTS_SPACES_ACCESS_KEY }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.RELEASE_TESTS_SPACES_SECRET_KEY }}
-      CODEXDOCKERIMAGE: ${{ needs.build-docker-dist-tests.outputs.logos_storage_image }}
+      STORAGEDOCKERIMAGE: ${{ needs.build-docker-dist-tests.outputs.logos_storage_image }}
       TEST_TYPE: release-tests
       BRANCH: ${{ inputs.branch || 'master' }}
       SOURCE: https://github.com/logos-storage/logos-storage-nim-cs-dist-tests
-      TF_DIR: .github/release/clusters/codex-dist-tests-do-ams3
+      TF_DIR: .github/release/clusters/logos-storage-dist-tests-do-ams3
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -241,7 +241,7 @@ jobs:
         run: terraform apply -auto-approve
 
       - name: Get kubeconfig
-        run: doctl kubernetes cluster kubeconfig save codex-dist-tests-do-ams3
+        run: doctl kubernetes cluster kubeconfig save logos-storage-dist-tests-do-ams3
 
       - name: Create in-cluster app kubeconfig secret
         run: |
@@ -254,7 +254,7 @@ jobs:
           export SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
           export CA=$(kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}')
 
-          kubectl create secret generic codex-dist-tests-app-kubeconfig \
+          kubectl create secret generic storage-dist-tests-app-kubeconfig \
             --from-file=kubeconfig.yaml=<(envsubst < .github/release/kubeconfig-template.yaml) \
             -n default
 

From 2e989e59efadf6fcd5639026190e90139e0ecc35 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Fri, 17 Apr 2026 15:25:56 +1000
Subject: [PATCH 13/14] Ensure the cluster is always destroyed

Always teardown the cluster if the terraform apply step has started to run, regardless of its result.
---
 .github/workflows/release.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 61aa7933..6d1e4ae4 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -237,6 +237,7 @@ jobs:
         run: terraform init
 
       - name: Terraform apply
+        id: tf-apply
         working-directory: ${{ env.TF_DIR }}
         run: terraform apply -auto-approve
 
@@ -291,7 +292,7 @@ jobs:
           [[ "$job_status" == "SuccessCriteriaMet" ]] || exit 1
 
       - name: Terraform destroy
-        if: always()
+        if: always() && steps.tf-apply.conclusion != 'skipped'
         working-directory: ${{ env.TF_DIR }}
         run: terraform destroy -auto-approve
 

From a6d364f889d04eeb3d27d4c9706135ddfffebc26 Mon Sep 17 00:00:00 2001
From: E M <5089238+emizzle@users.noreply.github.com>
Date: Mon, 20 Apr 2026 12:27:07 +1000
Subject: [PATCH 14/14] remove claude settings

---
 .claude/settings.json | 28 ----------------------------
 1 file changed, 28 deletions(-)
 delete mode 100644 .claude/settings.json

diff --git a/.claude/settings.json b/.claude/settings.json
deleted file mode 100644
index 9f0feb0f..00000000
--- a/.claude/settings.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Read(//Users/egonat/repos/gmega/logos-storage-docs/**)",
-      "Bash(npm run build)",
-      "Bash(npx astro build)",
-      "Bash(npm install)",
-      "Bash(node_modules/.bin/astro build)",
-      "WebSearch",
-      "Bash(npm test:*)",
-      "Bash(node -e \"import\\(''koffi''\\).then\\(m => console.log\\(Object.keys\\(m.default\\)\\)\\).catch\\(console.error\\)\")",
-      "Bash(timeout 5 node -e \":*)",
-      "Bash(node probe.js)",
-      "Bash(node two-node-test.js)",
-      "WebFetch(domain:github.com)",
-      "WebFetch(domain:api.github.com)",
-      "Read(//Users/egonat/repos/logos-storage/infra-codex/clusters/codex-dist-tests-do-ams3/**)",
-      "Read(//Users/egonat/repos/logos-storage/logos-storage-nim-cs-dist-tests/Tests/CodexReleaseTests/**)",
-      "Read(//Users/egonat/repos/logos-storage/logos-storage-nim-cs-dist-tests/**)",
-      "Bash(grep -E \"\\\\.cs$\")",
-      "Read(//Users/egonat/repos/logos-storage/infra-codex/clusters/modules/doks/**)"
-    ],
-    "additionalDirectories": [
-      "/Users/egonat/repos/gmega/logos-storage-docs/src/content/docs/api",
-      "/Users/egonat/repos/gmega/logos-storage-docs/src/content/docs/tutorials"
-    ]
-  }
-}