Merge branch 'master' into feat/unified-debug-api

.github/actions/nimbus-build-system/action.yml

@@ -41,6 +41,15 @@ runs:
     - name: Homebrew (macOS)
       if: inputs.os == 'macos'
       shell: ${{ inputs.shell }} {0}
+      env:
+        # The macOS runner image ships preinstalled, untrusted taps
+        # (aws/tap, azure/bicep) that make brew emit a "taps are not trusted"
+        # warning on every invocation. We only install the core `libomp`
+        # formula (always trusted), so opt into Homebrew's upcoming default
+        # of ignoring untrusted taps to silence the warning. Using the
+        # forward-compatible flag; HOMEBREW_NO_REQUIRE_TAP_TRUST is slated
+        # for removal.
+        HOMEBREW_REQUIRE_TAP_TRUST: 1
       run: |
         brew install libomp
 

.github/workflows/docker-reusable.yml

@@ -126,6 +126,11 @@ jobs:
     runs-on: ${{ matrix.builder }}
     env:
       PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+      # docker/build-push-action auto-uploads a build-record (.dockerbuild)
+      # artifact and a job summary by default. We don't consume either, so
+      # disable both to keep the workflow run free of stray artifacts.
+      DOCKER_BUILD_RECORD_UPLOAD: false
+      DOCKER_BUILD_SUMMARY: false
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -142,7 +147,7 @@ jobs:
       - name: Docker - Login to Docker Hub
         uses: docker/login-action@v4
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          username: ${{ vars.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker - Build and Push by digest
@@ -234,7 +239,7 @@ jobs:
       - name: Docker - Login to Docker Hub
         uses: docker/login-action@v4
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          username: ${{ vars.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker - Create manifest list and push

.github/workflows/release.yml

@@ -3,24 +3,24 @@ name: Release
 on:
   push:
     tags:
-      - 'v*.*.*'
+      - "v*.*.*"
     branches:
       - master
   workflow_dispatch:
     inputs:
       branch:
-        description: 'dist-tests branch to run tests from'
+        description: "dist-tests branch to run tests from"
         required: false
-        default: 'master'
+        default: "master"
         type: string
 
-concurrency: 
+concurrency:
   # Prenvent concurrent release runs. This happens, eg, in a
   # double-dispatch (clicking "Run workflow" multiple times in a row) or if
   # multiple commits are pushed in quick succession to master (eg, a bugfix
   # right after a release commit).
-  # All runs target the same shared GCP cluster/Terraform state, so serialize 
-  # them instead of racing for the state lock. 
+  # All runs target the same shared GCP cluster/Terraform state, so serialize
+  # them instead of racing for the state lock.
   group: ${{ github.workflow }}
   cancel-in-progress: false
 
@@ -30,8 +30,8 @@ env:
   storage_binary_base: logos-storage
   c_bindings_lib_base: libstorage
   build_dir: build
-  nim_flags: ''
-  windows_libs: 'libstdc++-6.dll libgomp-1.dll libgcc_s_seh-1.dll libwinpthread-1.dll'
+  nim_flags: ""
+  windows_libs: "libstdc++-6.dll libgomp-1.dll libgcc_s_seh-1.dll libwinpthread-1.dll"
 
 jobs:
   # Matrix
@@ -40,15 +40,15 @@ jobs:
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
     steps:
-    - name: Compute matrix
-      id: matrix
-      uses: fabiocaccamo/create-matrix-action@v5
-      with:
-        matrix: |
-          os {linux},   cpu {amd64}, builder {ubuntu-22.04},     nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-          os {linux},   cpu {arm64}, builder {ubuntu-22.04-arm}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-          os {macos},   cpu {arm64}, builder {macos-14},         nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
-          os {windows}, cpu {amd64}, builder {windows-latest},   nim_version {${{ env.nim_version }}}, shell {msys2}
+      - name: Compute matrix
+        id: matrix
+        uses: fabiocaccamo/create-matrix-action@v5
+        with:
+          matrix: |
+            os {linux},   cpu {amd64}, builder {ubuntu-24.04},     nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {linux},   cpu {arm64}, builder {ubuntu-24.04-arm}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {macos},   cpu {arm64}, builder {macos-14},         nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail}
+            os {windows}, cpu {amd64}, builder {windows-latest},   nim_version {${{ env.nim_version }}}, shell {msys2}
 
   # Build
   build:
@@ -104,16 +104,29 @@ jobs:
           echo "storage_binary=${storage_binary}" >>$GITHUB_ENV
           echo "c_bindings_lib=${c_bindings_lib}" >>$GITHUB_ENV
 
+      - name: Compute platform Nim flags
+        id: platform-nim-flags
+        shell: bash
+        run: |
+          platform_nim_flags=""
+          # secp256k1's x86_64 inline asm can hit a GCC register-allocation
+          # failure when -march=native expands against this runner's CPU, so
+          # pin a fixed baseline instead. arm64/macOS don't use that asm path.
+          if [[ "${{ matrix.os }}" == "linux" && "${{ matrix.cpu }}" == "amd64" ]]; then
+            platform_nim_flags="-d:disableMarchNative"
+          fi
+          echo "nim_flags=${platform_nim_flags}" >> "$GITHUB_OUTPUT"
+
       - name: Build Logos Storage binary
         run: |
-          make NIMFLAGS="--out:${{ env.build_dir }}/${{ env.storage_binary }} ${{ env.nim_flags }}"
+          make NIMFLAGS="${NIMFLAGS} --out:${{ env.build_dir }}/${{ env.storage_binary }} ${{ env.nim_flags }} ${{ steps.platform-nim-flags.outputs.nim_flags }}"
 
       - name: Package ${{ env.storage_binary_base }} Linux (compress and preserve perms)
         if: matrix.os == 'linux'
         run: |
           sudo apt-get update && sudo apt-get install -y zip
           zip -j "${{ env.build_dir }}/${{env.storage_binary}}.zip" ./${{ env.build_dir }}/*
-          
+
       - name: Package ${{ env.storage_binary_base }} MacOS (compress and preserve perms)
         if: matrix.os == 'macos'
         run: |
@@ -153,20 +166,20 @@ jobs:
         if: matrix.os == 'linux'
         run: |
           make -j${ncpu} update
-          make -j${ncpu} libstorage
+          make -j${ncpu} NIMFLAGS="${NIMFLAGS} ${{ steps.platform-nim-flags.outputs.nim_flags }}" libstorage
 
       - name: Build ${{ env.c_bindings_lib_base }} (MacOS)
         if: matrix.os == 'macos'
         run: |
           make -j${ncpu} update
-          STORAGE_LIB_PARAMS="--passL:\"-Wl,-install_name,@rpath/${{ env.c_bindings_lib_base }}.dylib\"" make -j${ncpu} libstorage
+          STORAGE_LIB_PARAMS="--passL:\"-Wl,-install_name,@rpath/${{ env.c_bindings_lib_base }}.dylib\"" make -j${ncpu} NIMFLAGS="${NIMFLAGS} ${{ steps.platform-nim-flags.outputs.nim_flags }}" libstorage
 
       - name: Build ${{ env.c_bindings_lib_base }} (Windows)
         if: matrix.os == 'windows'
         shell: msys2 {0}
         run: |
           make -j${ncpu} update
-          make -j${ncpu} libstorage
+          make -j${ncpu} NIMFLAGS="${NIMFLAGS} ${{ steps.platform-nim-flags.outputs.nim_flags }}" libstorage
 
       - name: Package ${{ env.c_bindings_lib_base }} Linux
         if: matrix.os == 'linux'
@@ -174,7 +187,7 @@ jobs:
           sudo apt-get update && sudo apt-get install -y zip
           zip -j "${{ env.build_dir }}/${{ env.c_bindings_lib }}.zip" ${{ env.build_dir }}/${{ env.c_bindings_lib_base }}.so
           zip -j "${{ env.build_dir }}/${{ env.c_bindings_lib }}.zip" library/${{ env.c_bindings_lib_base }}.h
-          
+
       - name: Package ${{ env.c_bindings_lib_base }} MacOS
         if: matrix.os == 'macos'
         run: |
@@ -201,7 +214,7 @@ jobs:
     if: github.ref_type == 'tag' || github.event_name == 'workflow_dispatch'
     uses: ./.github/workflows/docker-reusable.yml
     with:
-      nimflags: '-d:disableMarchNative -d:storage_enable_api_debug_peers=true -d:storage_enable_log_counter=true'
+      nimflags: "-d:disableMarchNative -d:storage_enable_api_debug_peers=true -d:storage_enable_log_counter=true"
       nat_ip_auto: true
       tag_latest: false
       tag_stable: false
@@ -224,7 +237,7 @@ jobs:
       TF_VAR_region: europe-west4
       TF_VAR_zone: europe-west4-a
       TF_PLUGIN_CACHE_DIR: /home/runner/.terraform.d/plugin-cache
-      STORAGEDOCKERIMAGE: ${{ github.ref_type == 'tag' && format('logosstorage/logos-storage-nim:{0}-dist-tests', github.ref_name) || 'logosstorage/logos-storage-nim:latest-dist-tests' }}
+      STORAGEDOCKERIMAGE: ${{ needs.build-docker-dist-tests.outputs.logos_storage_image }}
       TEST_TYPE: release-tests
       BRANCH: ${{ inputs.branch || 'master' }}
       SOURCE: https://github.com/logos-storage/logos-storage-nim-cs-dist-tests
@@ -424,14 +437,14 @@ jobs:
       - name: Download binaries from workflow artifacts into temp folder
         uses: actions/download-artifact@v8
         with:
-          pattern: ${{ env.storage_binary_base }}*
+          pattern: ${{ env.storage_binary_base }}*.zip
           merge-multiple: true
           path: /tmp/release
 
       - name: Download ${{ env.c_bindings_lib_base }} from workflow artifacts into temp folder
         uses: actions/download-artifact@v8
         with:
-          pattern: ${{ env.c_bindings_lib_base }}*
+          pattern: ${{ env.c_bindings_lib_base }}*.zip
           merge-multiple: true
           path: /tmp/release
 

build.nims

@@ -67,7 +67,7 @@ task storage, "build logos storage binary":
     params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
 
 task mixTools, "build mix tools (mix_pool, mix_relay_dht)":
-  let (desc, ec) = gorgeEx("git describe --always --dirty")
+  let (desc, ec) = gorgeEx("git describe --tags --always --dirty")
   let mixVersion =
     if ec == 0 and desc.strip().len > 0:
       desc.strip()

library/storage_thread_requests/requests/node_mix_request.nim

@@ -24,5 +24,6 @@ proc process*(
   defer:
     destroyShared(self)
 
-  let previous = storage[].node.togglePrivateQueries(self.privateQueries)
+  let previous = storage[].node.togglePrivateQueries(self.privateQueries).valueOr:
+    return err(error.msg)
   return ok($previous)

storage/discovery.nim

@@ -262,10 +262,12 @@ proc close*(d: Discovery) {.async: (raises: []).} =
   else:
     trace "Discovery store closed"
 
-proc togglePrivateQueries*(d: Discovery, enabled: bool): bool =
+proc togglePrivateQueries*(d: Discovery, enabled: bool): ?!bool =
+  if enabled and (d.mixProto.isNil or d.dhtMixProxies.len == 0):
+    return failure("Cannot enable private queries: Mix is not configured")
   let old = d.privateQueries
   d.privateQueries = enabled
-  return old
+  success(old)
 
 proc new*(
     T: type Discovery,

storage/node.nim

@@ -450,8 +450,8 @@ proc iterateManifests*(self: StorageNodeRef, onManifest: OnManifest) {.async.} =
 
       onManifest(cid, manifest)
 
-proc togglePrivateQueries*(self: StorageNodeRef, enable: bool): bool =
-  return self.discovery.togglePrivateQueries(enable)
+proc togglePrivateQueries*(self: StorageNodeRef, enable: bool): ?!bool =
+  self.discovery.togglePrivateQueries(enable)
 
 proc onExpiryUpdate(
     self: StorageNodeRef, rootCid: Cid, expiry: SecondsSince1970

storage/storage.nim

@@ -122,6 +122,11 @@ proc start*(s: StorageServer) {.async.} =
 
     s.storageNode.discovery.mixProto = mixProto
 
+    if s.config.dhtMixProxies.len > 0:
+      discard s.storageNode.discovery.togglePrivateQueries(true).valueOr:
+        raise
+          newException(StorageError, "Failed to enable private queries: " & error.msg)
+
     s.storageNode.engine.network.excludeRelays(relayPool.keys.toSeq)
 
   let (announceAddrs, discoveryAddrs) = nattedAddress(
@@ -375,11 +380,6 @@ proc new*(
   switch.mount(network)
   switch.mount(manifestProto)
 
-  # Enables private queries by default when mix is enabled.
-  if config.mixEnabled:
-    info "Enabling private queries over DHT by default", enabled = config.mixEnabled
-    discard discovery.togglePrivateQueries(true)
-
   StorageServer(
     config: config,
     storageNode: storageNode,

tests/cbindings/storage.c

@@ -869,9 +869,9 @@ int check_toggle_private_queries(void *storage_ctx)
     }
 
     int ret = is_resp_ok(r, &res);
-    if (res == NULL || strcmp(res, "false") != 0)
+    if (ret == RET_OK)
     {
-        fprintf(stderr, "toggle private queries content mismatch, res:%s\n", res ? res : "(null)");
+        fprintf(stderr, "expected toggle(true) to fail when mix is not configured, got ok\n");
         free(res);
         return RET_ERR;
     }
@@ -886,7 +886,7 @@ int check_toggle_private_queries(void *storage_ctx)
     }
 
     ret = is_resp_ok(r, &res);
-    if (res == NULL || strcmp(res, "true") != 0)
+    if (res == NULL || strcmp(res, "false") != 0)
     {
         fprintf(stderr, "toggle private queries content mismatch, res:%s\n", res ? res : "(null)");
         free(res);

tests/storage/blockexchange/discovery/testdiscovery.nim

@@ -151,5 +151,13 @@ asyncchecksuite "Block Advertising and Discovery":
     discovery.refCid = refCid
 
     check (await discovery.find(refCid)) == @[directSpr]
-    check discovery.togglePrivateQueries(true) == false
+    let toggleRes = discovery.togglePrivateQueries(true)
+    check toggleRes.isOk
+    check toggleRes.get == false
     check (await discovery.find(refCid)) == @[privateSpr]
+
+  test "should fail to enable private queries when MixProtocol is nil":
+    let discovery = MixMockDiscovery.new()
+    discovery.dhtMixProxies = @[SignedPeerRecord.example]
+    let res = discovery.togglePrivateQueries(true)
+    check res.isErr