Merge pull request #19 from logos-storage/build/dev-env-gotestsum-and-makefile

Build: small updates to dev env: debugging, gotestsum and makefile

.github/workflows/ci.yml

@@ -46,5 +46,8 @@ jobs:
       - name: Build codex go
         run: make
 
+      - name: Install gotestsum
+        run: go install gotest.tools/gotestsum@latest
+
       - name: Go test
         run: make test

.vscode/launch.json

@@ -0,0 +1,77 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug Current Test",
+      "type": "go",
+      "request": "launch",
+      "mode": "test",
+      "program": "${fileDirname}",
+      "args": ["-test.v", "-test.run", "^${selectedText}$"],
+      "env": {
+        "CGO_ENABLED": "1"
+      }
+    },
+    {
+      "name": "Debug Current Test Function",
+      "type": "go",
+      "request": "launch",
+      "mode": "test",
+      "program": "${fileDirname}",
+      "env": {
+        "CGO_ENABLED": "1"
+      }
+    },
+    {
+      "name": "Debug All Tests in Current File",
+      "type": "go",
+      "request": "launch",
+      "mode": "test",
+      "program": "${fileDirname}",
+      "args": ["-test.v"],
+      "env": {
+        "CGO_ENABLED": "1"
+      }
+    },
+    {
+      "name": "Debug All Tests in Current Package",
+      "type": "go",
+      "request": "launch",
+      "mode": "test",
+      "program": "${fileDirname}",
+      "args": ["-test.v", "-count=1"],
+      "env": {
+        "CGO_ENABLED": "1"
+      }
+    },
+    {
+      "name": "Debug Codex Tests",
+      "type": "go",
+      "request": "launch",
+      "mode": "test",
+      "program": "${workspaceFolder}/codex",
+      "args": ["-test.v"],
+      "env": {
+        "CGO_ENABLED": "1"
+      }
+    },
+    {
+      "name": "Debug Specific Test (e.g., TestUpload)",
+      "type": "go",
+      "request": "launch",
+      "mode": "test",
+      "program": "${workspaceFolder}/codex",
+      "args": [
+        "-test.v",
+        "-test.run",
+        "TestUpload"
+      ],
+      "env": {
+        "CGO_ENABLED": "1"
+      }
+    }
+  ]
+}

CHANGELOG.md

@@ -1,3 +1,36 @@
+## v0.0.28 (2025-11-14)
+### Notes
+
+- fix: bump nim codex to prevent datastore lock when closing the Codex client
+- fix: configuration duration for block-ttl, block-mi and int for num-threads
+
+## v0.0.27 (2025-11-11)
+### Notes
+
+- Enhance release note in
+- Bump nim-codex to prototype release branch
+
+## v0.0.26 (2025-11-03)
+### Notes
+
+-  Bump `nim-codex` to prototype release branch
+
+## v0.0.25 (2025-11-03)
+### Notes
+
+- Add `exists` to check the existence of a cid in the local store
+
+## v0.0.24 (2025-10-30)
+### Notes
+
+- Return the standard context.Canceled when a context is cancelled
+
+## v0.0.23 (2025-10-30)
+### Notes
+
+- Add context cancellation support
+- Prevent segmentation fault when trying to stop and node not started
+
 ## v0.0.22 (2025-10-20)
 ### Notes
 

Makefile

@@ -23,13 +23,24 @@ libcodex:
 	@echo "Building libcodex..."
 	@$(MAKE) -C $(NIM_CODEX_DIR) libcodex
 
+libcodex-with-debug-api:
+	@echo "Building libcodex..."
+	@$(MAKE) -C $(NIM_CODEX_DIR) libcodex CODEX_LIB_PARAMS="-d:codex_enable_api_debug_peers"
+
 build:
 	@echo "Building Codex Go Bindings..."
 	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o codex-go ./codex
 
 test:
 	@echo "Running tests..."
-	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOTESTFLAGS="-timeout=2m" go test ./...
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOTESTFLAGS="-timeout=2m" gotestsum --packages="./..." -f testname -- $(if $(filter-out test,$(MAKECMDGOALS)),-run "$(filter-out test,$(MAKECMDGOALS))")
+
+test-with-params:
+	@echo "Running tests..."
+	CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOTESTFLAGS="-timeout=2m" gotestsum --packages="./..." -f testname -- $(ARGS)
+
+%:
+	@:
 
 clean:
 	@echo "Cleaning up..."

README.md

@@ -78,22 +78,63 @@ Follow these steps to install and set up the module:
 1. Make sure your system has the [prerequisites](https://github.com/codex-storage/nim-codex) to run a local Codex node.
 
 2. Fetch the dependencies:
-   ```
+   ```sh
    make update
    ```
 
 3. Build the library:
-   ```
+   ```sh
    make libcodex
    ```
 
 You can pass flags to the Codex building step by using `CODEX_LIB_PARAMS`. For example,
 if you want to enable debug API for peers, you can build the library using:
 
-```
+```sh
 CODEX_LIB_PARAMS="-d:codex_enable_api_debug_peers=true" make libcodex
 ```
 
+or you can use a convenience `libcodex-with-debug-api` make target:
+
+```sh
+make libcodex-with-debug-api
+```
+
+To run the test, you have to make sure you have `gotestsum` installed on your system, e.g.:
+
+```sh
+go install gotest.tools/gotestsum@latest
+```
+
+Then you can run the tests as follows.
+
+To run all the tests:
+
+```sh
+make test
+```
+
+To run selected test only:
+
+```sh
+make test "TestDownloadManifest$"
+```
+
+> We use `$` to make sure we run only the `TestDownloadManifest` test.
+> Without `$` we would run all the tests starting with `TestDownloadManifest` and
+> so also `TestDownloadManifestWithNotExistingCid`
+>
+
+If you need to pass more arguments to the underlying `go test` (`gotestsum` passes
+everything after `--` to `go test`), you can use: `test-with-params` make target, e.g.:
+
+```sh
+make test-with-params ARGS='-run "TestDownloadManifest$$" -count=2'
+```
+
+> Here, we use double escape `$$` instead of just `$`, otherwise make
+> will interpret `$` as a make variable inside `ARGS`. 
+
 Now the module is ready for use in your project.
 
 The release process is defined [here](./RELEASE.md).

RELEASE.md

@@ -27,3 +27,70 @@ Once published, the artifacts can be downloaded using  the `version`, example:
 
 It is not recommended to use the `latest` URL because you may face cache issues.
 
+## Integration
+
+Once released, you can integrate it into your Go project using:
+
+```bash
+go get github.com/codex-storage/codex-go-bindings@v0.0.26
+```
+
+Then you can use the following `Makefile` command to fetch the artifact:
+
+```bash
+LIBS_DIR := $(abspath ./libs)
+CODEX_OS := linux
+CODEX_ARCH := amd64
+CODEX_VERSION := $(shell go list -m -f '{{.Version}}' github.com/codex-storage/codex-go-bindings 2>/dev/null)
+CODEX_DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(CODEX_VERSION)/codex-${CODEX_OS}-${CODEX_ARCH}.zip"
+
+fetch-libcodex:
+    mkdir -p $(LIBS_DIR); \
+    curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${CODEX_OS}-${CODEX_ARCH}.zip ${CODEX_DOWNLOAD_URL}; \
+    unzip -o -qq $(LIBS_DIR)/codex-${CODEX_OS}-${CODEX_ARCH}.zip -d $(LIBS_DIR); \
+    rm -f $(LIBS_DIR)/codex*.zip;
+```
+
+`CODEX_VERSION` uses the same version as the Codex Go dependency declared in your project.
+
+### Nix
+
+If you use Nix in a sandboxed environment, you cannot use curl to download the artifacts, so you have to prefetch them using the artifacts `SHA-256` hash. To generate the hash, you can use the following command: 
+
+```bash
+nix store prefetch-file --json --unpack https://github.com/codex-storage/codex-go-bindings/releases/download/v0.0.26/codex-macos-arm64.zip | jq -r .hash
+
+# [10.4 MiB DL] sha256-3CHIWoSjo0plsYqzXQWm1EtY1STcljV4yfXTPon90uE=
+```
+
+Then include this hash in your Nix configuration. For example:
+
+```nix
+let
+  optionalString = pkgs.lib.optionalString;
+  codexVersion = "v0.0.26";
+  arch =
+    if stdenv.hostPlatform.isx86_64 then "amd64"
+    else if stdenv.hostPlatform.isAarch64 then "arm64"
+    else stdenv.hostPlatform.arch;
+  os = if stdenv.isDarwin then "macos" else "Linux";
+  hash =
+    if stdenv.hostPlatform.isDarwin
+    # nix store prefetch-file --json --unpack https://github.com/codex-storage/codex-go-bindings/releases/download/v0.0.26/codex-macos-arm64.zip | jq -r .hash
+    then "sha256-3CHIWoSjo0plsYqzXQWm1EtY1STcljV4yfXTPon90uE="
+    # nix store prefetch-file --json --unpack https://github.com/codex-storage/codex-go-bindings/releases/download/v0.0.26/codex-Linux-amd64.zip | jq -r .hash
+    else "sha256-YxW2vFZlcLrOx1PYgWW4MIstH/oFBRF0ooS0sl3v6ig=";
+
+  # Pre-fetch libcodex to avoid network during build
+  codexLib = pkgs.fetchzip {
+    url = "https://github.com/codex-storage/codex-go-bindings/releases/download/${codexVersion}/codex-${os}-${arch}.zip";
+    hash = hash;
+    stripRoot = false;
+  };
+
+  preBuild = ''
+    export LIBS_DIR="${codexLib}"
+    # Build something cool with Codex
+  '';
+```
+

codex/codex.go

@@ -159,12 +159,12 @@ type Config struct {
 
 	// Default block timeout in seconds - 0 disables the ttl
 	// Default: 30 days
-	BlockTtl int `json:"block-ttl,omitempty"`
+	BlockTtl string `json:"block-ttl,omitempty"`
 
 	// Time interval in seconds - determines frequency of block
 	// maintenance cycle: how often blocks are checked for expiration and cleanup
 	// Default: 10 minutes
-	BlockMaintenanceInterval int `json:"block-mi,omitempty"`
+	BlockMaintenanceInterval string `json:"block-mi,omitempty"`
 
 	// Number of blocks to check every maintenance cycle
 	// Default: 1000

codex/codex_test.go

@@ -1,6 +1,8 @@
 package codex
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestCodexVersion(t *testing.T) {
 	config := defaultConfigHelper(t)
@@ -79,3 +81,74 @@ func TestPeerId(t *testing.T) {
 
 	t.Logf("Codex PeerId: %s", peerId)
 }
+
+func TestStorageQuota(t *testing.T) {
+	node := newCodexNode(t, Config{
+		StorageQuota: 1024 * 1024 * 1024, // 1GB
+	})
+
+	if node == nil {
+		t.Fatal("expected codex node to be created")
+	}
+}
+
+func TestCreateAndDestroyMultipleInstancesWithSameDatadir(t *testing.T) {
+	datadir := t.TempDir()
+
+	config := Config{
+		DataDir:        datadir,
+		LogFormat:      LogFormatNoColors,
+		MetricsEnabled: false,
+		BlockRetries:   5,
+		Nat:            "none",
+	}
+
+	for range 2 {
+		node, err := New(config)
+		if err != nil {
+			t.Fatalf("Failed to create Codex node: %v", err)
+		}
+
+		if err := node.Start(); err != nil {
+			t.Fatalf("Failed to start Codex node: %v", err)
+		}
+
+		if err := node.Stop(); err != nil {
+			t.Fatalf("Failed to stop Codex node: %v", err)
+		}
+
+		if err := node.Destroy(); err != nil {
+			t.Fatalf("Failed to stop Codex node after restart: %v", err)
+		}
+	}
+}
+
+func TestNumThreads(t *testing.T) {
+	node := newCodexNode(t, Config{
+		NumThreads: 1,
+	})
+
+	if node == nil {
+		t.Fatal("expected codex node to be created")
+	}
+}
+
+func TestBlockTtl(t *testing.T) {
+	node := newCodexNode(t, Config{
+		BlockTtl: "10H",
+	})
+
+	if node == nil {
+		t.Fatal("expected codex node to be created")
+	}
+}
+
+func TestBlockMaintenanceInterval(t *testing.T) {
+	node := newCodexNode(t, Config{
+		BlockMaintenanceInterval: "10H",
+	})
+
+	if node == nil {
+		t.Fatal("expected codex node to be created")
+	}
+}

codex/p2p_test.go

@@ -36,6 +36,7 @@ func TestConnectWithAddress(t *testing.T) {
 		LogFormat:      LogFormatNoColors,
 		MetricsEnabled: false,
 		DiscoveryPort:  8090,
+		Nat:            "none",
 	})
 	if err != nil {
 		t.Fatalf("Failed to create codex1: %v", err)

codex/storage.go

@@ -24,6 +24,10 @@ import (
    static int cGoCodexStorageDelete(void* codexCtx, char* cid, void* resp) {
        return codex_storage_delete(codexCtx, cid, (CodexCallback) callback, resp);
    }
+
+   static int cGoCodexStorageExists(void* codexCtx, char* cid, void* resp) {
+       return codex_storage_exists(codexCtx, cid, (CodexCallback) callback, resp);
+   }
 */
 import "C"
 
@@ -142,3 +146,19 @@ func (node CodexNode) Delete(cid string) error {
 	_, err := bridge.wait()
 	return err
 }
+
+// Exists checks if a given cid exists in the local storage.
+func (node CodexNode) Exists(cid string) (bool, error) {
+	bridge := newBridgeCtx()
+	defer bridge.free()
+
+	var cCid = C.CString(cid)
+	defer C.free(unsafe.Pointer(cCid))
+
+	if C.cGoCodexStorageExists(node.ctx, cCid, bridge.resp) != C.RET_OK {
+		return false, bridge.callError("cGoCodexStorageExists")
+	}
+
+	result, err := bridge.wait()
+	return result == "true", err
+}

codex/storage_test.go

@@ -119,3 +119,30 @@ func TestDelete(t *testing.T) {
 		t.Fatal("expected manifests to be empty after deletion")
 	}
 }
+
+func TestExists(t *testing.T) {
+	codex := newCodexNode(t)
+
+	cid, _ := uploadHelper(t, codex)
+
+	exists, err := codex.Exists(cid)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !exists {
+		t.Fatal("expected cid to exist")
+	}
+
+	err = codex.Delete(cid)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	exists, err = codex.Exists(cid)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if exists {
+		t.Fatal("expected cid to not exist after deletion")
+	}
+}

codex/testutil.go

@@ -14,7 +14,7 @@ func defaultConfigHelper(t *testing.T) Config {
 		LogFormat:      LogFormatNoColors,
 		MetricsEnabled: false,
 		BlockRetries:   3000,
-		LogLevel:       "ERROR",
+		Nat:            "none",
 	}
 }
 
@@ -43,6 +43,22 @@ func newCodexNode(t *testing.T, opts ...Config) *CodexNode {
 		if c.DiscoveryPort != 0 {
 			config.DiscoveryPort = c.DiscoveryPort
 		}
+
+		if c.StorageQuota != 0 {
+			config.StorageQuota = c.StorageQuota
+		}
+
+		if c.NumThreads != 0 {
+			config.NumThreads = c.NumThreads
+		}
+
+		if c.BlockTtl != "" {
+			config.BlockTtl = c.BlockTtl
+		}
+
+		if c.BlockMaintenanceInterval != "" {
+			config.BlockMaintenanceInterval = c.BlockMaintenanceInterval
+		}
 	}
 
 	node, err := New(config)

vendor/nim-codex

@@ -1 +1 @@
-Subproject commit a86d8586456d5eb6805b228e80e264ee736a6a90
+Subproject commit 2b9fc1eb554e5eee43b8a815084fb8c61687ada9