logos-storage/logos-storage-go
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-go.git synced 2026-01-02 13:23:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add codex library integration

Browse Source
This commit is contained in:
Arnaud 2025-10-28 06:56:13 +01:00
parent 0c4d537303
commit a74ac84124
No known key found for this signature in database
GPG Key ID: 20E40A5D3110766F
21 changed files with 424 additions and 938 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -21,3 +21,5 @@ coverage*.cov
# Logs
*.log
libs

11
.vscode/settings.json vendored
View File

@ -1,6 +1,13 @@
{
"go.testTags": "codex_integration",
"gopls": {
"buildFlags": ["-tags=codex_integration"]
"buildFlags": [
"-tags=integration"
]
},
"go.toolsEnvVars": {
"CGO_ENABLED": "1",
"CGO_CFLAGS": "-I${workspaceFolder}/libs",
"CGO_LDFLAGS": "-L${workspaceFolder}/libs -lcodex -Wl,-rpath,${workspaceFolder}/libs"
}
}
}

46
Makefile Normal file
View File

@ -0,0 +1,46 @@
# Destination folder for the downloaded libraries
LIBS_DIR := $(abspath ./libs)
# Flags for CGO to find the headers and the shared library
UNAME_S := $(shell uname -s)
CGO_CFLAGS := -I$(LIBS_DIR)
CGO_LDFLAGS := -L$(LIBS_DIR) -lcodex -Wl,-rpath,$(LIBS_DIR)
ifeq ($(OS),Windows_NT)
BIN_NAME := codex-go.exe
else
BIN_NAME := codex-go
endif
# Configuration for fetching the right binary
OS ?= "linux"
ARCH ?= "amd64"
VERSION ?= "v0.0.22"
DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(VERSION)/codex-${OS}-${ARCH}.zip"
fetch:
@echo "Fetching libcodex from GitHub Actions from: ${DOWNLOAD_URL}"
curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${OS}-${ARCH}.zip ${DOWNLOAD_URL}
unzip -o -qq $(LIBS_DIR)/codex-${OS}-${ARCH}.zip -d $(LIBS_DIR)
rm -f $(LIBS_DIR)/*.zip
build:
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o $(BIN_NAME) main.go
build-upload:
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-upload ./cmd/upload
build-download:
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-download ./cmd/download
test:
@echo "Running unit tests..."
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -v ./communities
test-integration:
@echo "Running tests..."
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -v -tags=codex_integration ./communities -run Integration -timeout 15s
clean:
rm -f $(BIN_NAME)
rm -Rf $(LIBS_DIR)/*

147
README.md
View File

@ -10,25 +10,12 @@ A lightweight Go client utility for interacting with Codex client.
We will be running codex client, and then use a small testing utility to check if the low level abstraction - CodexClient - correctly uploads and downloads the content.
### Running CodexClient
### Integration Codex library
I often remove some logging noise, by slightly changing the build
params in `build.nims` (nim-codex):
You need to download the library file by using:
```nim
task codex, "build codex binary":
buildBinary "codex",
# params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
params =
"-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE"
```
You see a slightly more selective `params` in the `codex` task.
To run the client I use the following command:
```bash
./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE
```sh
make fetch
```
### Building codex-upload and codex-download utilities
@ -36,8 +23,8 @@ To run the client I use the following command:
Use the following command to build the `codex-upload` and `codex-download` utilities:
```bash
go build -o bin/codex-upload ./cmd/upload
go build -o bin/codex-download ./cmd/download
make build-upload
make build-download
```
### Uploading content to Codex
@ -45,8 +32,8 @@ Now, using the `codex-upload` utility, we can upload the content to Codex as fol
```bash
~/code/local/go-codex-client
❯ ./bin/codex-upload -file test-data.bin -host localhost -port 8001
Uploading test-data.bin (43 bytes) to Codex at localhost:8001...
❯ ./bin/codex-upload -file test-data.bin
Uploading test-data.bin (43 bytes) to Codex
✅ Upload successful!
CID: zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V
```
@ -57,8 +44,8 @@ Now, having the content uploaded to Codex - let's get it back using the `codex-d
```bash
~/code/local/go-codex-client
❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin -host localhost -port 8001
Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex at localhost:8001...
❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin
Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex...
✅ Download successful!
Saved to: output.bin
```
@ -85,115 +72,23 @@ next section.
To run all unit tests:
```bash
❯ go test -v ./communities -count 1
❯ make test
=== RUN TestUpload_Success
--- PASS: TestUpload_Success (0.00s)
=== RUN TestDownload_Success
--- PASS: TestDownload_Success (0.00s)
=== RUN TestDownloadWithContext_Cancel
--- PASS: TestDownloadWithContext_Cancel (0.04s)
PASS
ok go-codex-client/communities 0.044s
```
To be more selective, e.g. in order to run all the tests from
`CodexArchiveDownloaderSuite`, run:
To run the integration test, use `test-integration`:
```bash
go test -v ./communities -run CodexArchiveDownloader -count 1
make test-integration
```
or for an individual test from that suite:
```bash
go test -v ./communities -run TestCodexArchiveDownloaderSuite/TestCancellationDuringPolling -count 1
```
You can also use `gotestsum` to run the tests (you may need to install it first, e.g. `go install gotest.tools/gotestsum@v1.13.0`):
```bash
gotestsum --packages="./communities" -f testname --rerun-fails -- -count 1
```
For a more verbose output including logs use `-f standard-verbose`, e.g.:
```bash
gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -v -count 1
```
To be more selective, e.g. in order to run all the tests from
`CodexArchiveDownloaderSuite`, run:
```bash
gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArchiveDownloader -count 1
```
or for an individual test from that suite:
```bash
gotestsum --packages="./communities" -f testname --rerun-fails -- -run TestCodexArchiveDownloaderSuite/TestCancellationDuringPolling -count 1
```
Notice, that the `-run` flag accepts a regular expression that matches against the full test path, so you can be more concise in naming if necessary, e.g.:
```bash
gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArchiveDownloader/Cancellation -count 1
```
This also applies to native `go test` command.
### Running integration tests
When building Codex client for testing like here, I often remove some logging noise, by slightly changing the build params in `build.nims`:
```nim
task codex, "build codex binary":
buildBinary "codex",
# params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
params =
"-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE"
```
You see a slightly more selective `params` in the `codex` task.
To start Codex client, use e.g.:
```bash
./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE
```
To run the integration test, use `codex_integration` tag and narrow the scope using `-run Integration`:
```bash
CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s
```
This will run all integration tests, including CodexClient integration tests.
To make sure that the test is actually run and not cached, use `count` option:
```bash
CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s -count 1
```
To be more specific and only run the tests related to, e.g. index downloader or archive
downloader you can use:
```bash
CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexIndexDownloaderIntegration -timeout 15s -count 1
CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexArchiveDownloaderIntegration -timeout 15s -count 1
```
and then, if you prefer to use `gotestsum`:
```bash
CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexIndexDownloaderIntegration -v -count 1
CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexArchiveDownloaderIntegration -v -count 1
```
or to run all integration tests (including CodexClient integration tests):
```bash
CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -v -count 1 -run Integration
```
I prefer to be more selective when running integration tests.
### Regenerating artifacts
Everything you need comes included in the repo. But if you decide to change things,

29
cmd/download/main.go
View File

@ -5,14 +5,15 @@ import (
"fmt"
"log"
"os"
"path"
"go-codex-client/communities" // Import the local communities package
"github.com/codex-storage/codex-go-bindings/codex"
)
func main() {
var (
host = flag.String("host", "localhost", "Codex host")
port = flag.String("port", "8080", "Codex port")
cid = flag.String("cid", "", "CID of the file to download")
file = flag.String("file", "downloaded-file.bin", "File to save the downloaded data")
)
@ -24,7 +25,20 @@ func main() {
}
// Create Codex client
client := communities.NewCodexClient(*host, *port)
client, err := communities.NewCodexClient(codex.Config{
LogFormat: codex.LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
LogLevel: "ERROR",
DataDir: path.Join(os.TempDir(), "codex-client-data"),
})
if err != nil {
log.Fatalf("Failed to create CodexClient: %v", err)
}
if err := client.Start(); err != nil {
log.Fatalf("Failed to start CodexClient: %v", err)
}
// Create output file
outputFile, err := os.Create(*file)
@ -33,8 +47,6 @@ func main() {
}
defer outputFile.Close()
fmt.Printf("Downloading CID %s from Codex at %s:%s...\n", *cid, *host, *port)
// Download data - pass the io.Writer (outputFile), not the string
err = client.Download(*cid, outputFile)
if err != nil {
@ -43,6 +55,13 @@ func main() {
log.Fatalf("Download failed: %v", err)
}
if err := client.Stop(); err != nil {
log.Printf("Warning: Failed to stop CodexClient: %v", err)
}
if err := client.Destroy(); err != nil {
log.Printf("Warning: Failed to stop CodexClient: %v", err)
}
fmt.Printf("✅ Download successful!\n")
fmt.Printf("Saved to: %s\n", *file)
}

29
cmd/upload/main.go
View File

@ -6,14 +6,15 @@ import (
"fmt"
"log"
"os"
"path"
"go-codex-client/communities" // Import the local communities package
"github.com/codex-storage/codex-go-bindings/codex"
)
func main() {
var (
host = flag.String("host", "localhost", "Codex host")
port = flag.String("port", "8080", "Codex port")
file = flag.String("file", "test-data.bin", "File to upload")
filename = flag.String("name", "", "Filename to use in upload (defaults to actual filename)")
)
@ -31,15 +32,35 @@ func main() {
uploadName = *file
}
fmt.Printf("Uploading %s (%d bytes) to Codex at %s:%s...\n", *file, len(data), *host, *port)
fmt.Printf("Uploading %s (%d bytes) to Codex...\n", *file, len(data))
// Create Codex client and upload
client := communities.NewCodexClient(*host, *port)
client, err := communities.NewCodexClient(codex.Config{
LogFormat: codex.LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
LogLevel: "ERROR",
DataDir: path.Join(os.TempDir(), "codex-client-data"),
})
if err != nil {
log.Fatalf("Failed to create CodexClient: %v", err)
}
if err := client.Start(); err != nil {
log.Fatalf("Failed to start CodexClient: %v", err)
}
cid, err := client.Upload(bytes.NewReader(data), uploadName)
if err != nil {
log.Fatalf("Upload failed: %v", err)
}
if err := client.Stop(); err != nil {
log.Printf("Warning: Failed to stop CodexClient: %v", err)
}
if err := client.Destroy(); err != nil {
log.Printf("Warning: Failed to stop CodexClient: %v", err)
}
fmt.Printf("✅ Upload successful!\n")
fmt.Printf("CID: %s\n", cid)
}

4
communities/codex_archive_downloader.go
View File

@ -316,8 +316,8 @@ func (d *CodexArchiveDownloader) triggerSingleArchiveDownload(hash, cid string,
return fmt.Errorf("failed to trigger archive download with CID %s: %w", cid, err)
}
if manifest.CID != cid {
return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.CID, cid)
if manifest.Cid != cid {
return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.Cid, cid)
}
return nil

23
communities/codex_archive_downloader_integration_test.go
View File

@ -8,10 +8,10 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"os"
"testing"
"time"
"github.com/codex-storage/codex-go-bindings/codex"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -31,19 +31,18 @@ type CodexArchiveDownloaderIntegrationSuite struct {
// SetupSuite runs once before all tests in the suite
func (suite *CodexArchiveDownloaderIntegrationSuite) SetupSuite() {
// Use port 8001 as specified by the user
host := communities.GetEnvOrDefault("CODEX_HOST", "localhost")
port := communities.GetEnvOrDefault("CODEX_API_PORT", "8001")
suite.client = communities.NewCodexClient(host, port)
// Optional request timeout override
if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" {
if d, err := time.ParseDuration(ms + "ms"); err == nil {
suite.client.SetRequestTimeout(d)
}
var err error
suite.client, err = communities.NewCodexClient(codex.Config{
LogFormat: codex.LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
LogLevel: "ERROR",
})
if err != nil {
suite.T().Fatalf("Failed to create CodexClient: %v", err)
}
suite.T().Logf("CodexClient configured for %s:%s", host, port)
suite.T().Logf("CodexClient configured for")
}
// TearDownSuite runs once after all tests in the suite

45
communities/codex_archive_downloader_test.go
View File

@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/codex-storage/codex-go-bindings/codex"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -63,7 +64,7 @@ func (suite *CodexArchiveDownloaderSuite) TestBasicSingleArchive() {
// Set up mock expectations - same as before
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
Return(&communities.CodexManifest{CID: "test-cid-1"}, nil).
Return(codex.Manifest{Cid: "test-cid-1"}, nil).
Times(1)
// First HasCid call returns false, second returns true (simulating polling)
@ -149,7 +150,7 @@ func (suite *CodexArchiveDownloaderSuite) TestMultipleArchives() {
for _, cid := range expectedCids {
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), cid).
Return(&communities.CodexManifest{CID: cid}, nil).
Return(codex.Manifest{Cid: cid}, nil).
Times(1)
// Each archive becomes available after one poll
@ -236,7 +237,7 @@ func (suite *CodexArchiveDownloaderSuite) TestErrorDuringTriggerDownload() {
// Mock TriggerDownloadWithContext to simulate an error
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
Return(nil, assert.AnError). // Return a generic error to simulate failure
Return(codex.Manifest{}, assert.AnError). // Return a generic error to simulate failure
Times(1)
// No HasCid calls should be made since TriggerDownload fails
@ -288,13 +289,13 @@ func (suite *CodexArchiveDownloaderSuite) TestActualCancellationDuringTriggerDow
// Use DoAndReturn to create a realistic TriggerDownload that waits for cancellation
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) {
DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
// Simulate work by waiting for context cancellation
select {
case <-time.After(5 * time.Second): // This should never happen in our test
return &communities.CodexManifest{CID: cid}, nil
return codex.Manifest{Cid: cid}, nil
case <-ctx.Done(): // Wait for actual context cancellation
return nil, ctx.Err() // Return the actual cancellation error
return codex.Manifest{}, ctx.Err() // Return the actual cancellation error
}
}).
Times(1)
@ -352,7 +353,7 @@ func (suite *CodexArchiveDownloaderSuite) TestCancellationDuringPolling() {
// Mock successful TriggerDownload
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
Return(&communities.CodexManifest{CID: "test-cid-1"}, nil).
Return(codex.Manifest{Cid: "test-cid-1"}, nil).
Times(1)
// Mock polling - allow multiple calls, but we'll cancel before completion
@ -420,7 +421,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPollingTimeout() {
// Mock successful TriggerDownload
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
Return(&communities.CodexManifest{CID: "test-cid-1"}, nil).
Return(codex.Manifest{Cid: "test-cid-1"}, nil).
Times(1)
// Mock polling to always return false (simulating timeout)
@ -496,7 +497,7 @@ func (suite *CodexArchiveDownloaderSuite) TestWithExistingArchives() {
// Only archive-2 should be downloaded (not in existingArchiveIDs)
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(&communities.CodexManifest{CID: "cid-2"}, nil).
Return(codex.Manifest{Cid: "cid-2"}, nil).
Times(1) // Only one call expected
// Only archive-2 should be polled
@ -577,7 +578,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_OneSuccessOneError(
// Archive-2 succeeds
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(&communities.CodexManifest{CID: "cid-2"}, nil)
Return(codex.Manifest{Cid: "cid-2"}, nil)
suite.mockClient.EXPECT().
HasCid("cid-2").
Return(true, nil)
@ -585,7 +586,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_OneSuccessOneError(
// Archive-1 fails
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-1").
Return(nil, fmt.Errorf("trigger failed"))
Return(codex.Manifest{}, fmt.Errorf("trigger failed"))
logger := zap.NewNop()
downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger)
@ -633,7 +634,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessErrorCancell
// Archive-3 (newest) succeeds
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-3").
Return(&communities.CodexManifest{CID: "cid-3"}, nil)
Return(codex.Manifest{Cid: "cid-3"}, nil)
suite.mockClient.EXPECT().
HasCid("cid-3").
Return(true, nil)
@ -641,14 +642,14 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessErrorCancell
// Archive-2 fails
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(nil, fmt.Errorf("trigger failed"))
Return(codex.Manifest{}, fmt.Errorf("trigger failed"))
// Archive-1 will be cancelled (no expectations needed)
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-1").
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) {
DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
<-ctx.Done() // Wait for cancellation
return nil, ctx.Err()
return codex.Manifest{}, ctx.Err()
}).
AnyTimes()
@ -700,7 +701,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessThenCancella
// Archive-2 (newer) succeeds
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(&communities.CodexManifest{CID: "cid-2"}, nil)
Return(codex.Manifest{Cid: "cid-2"}, nil)
suite.mockClient.EXPECT().
HasCid("cid-2").
Return(true, nil)
@ -708,9 +709,9 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessThenCancella
// Archive-1 will be cancelled
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-1").
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) {
DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
<-ctx.Done() // Wait for cancellation
return nil, ctx.Err()
return codex.Manifest{}, ctx.Err()
}).
AnyTimes()
@ -762,9 +763,9 @@ func (suite *CodexArchiveDownloaderSuite) TestNoSuccess_OnlyCancellation() {
// Both archives will be cancelled
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) {
DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
<-ctx.Done() // Wait for cancellation
return nil, ctx.Err()
return codex.Manifest{}, ctx.Err()
}).
AnyTimes()
@ -815,10 +816,10 @@ func (suite *CodexArchiveDownloaderSuite) TestNoSuccess_OnlyErrors() {
// Both archives fail
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-1").
Return(nil, fmt.Errorf("trigger failed for cid-1"))
Return(codex.Manifest{}, fmt.Errorf("trigger failed for cid-1"))
suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(nil, fmt.Errorf("trigger failed for cid-2"))
Return(codex.Manifest{}, fmt.Errorf("trigger failed for cid-2"))
logger := zap.NewNop()
downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger)

261
communities/codex_client.go
View File

@ -9,63 +9,48 @@ package communities
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/codex-storage/codex-go-bindings/codex"
)
// CodexClient handles basic upload/download operations with Codex storage
type CodexClient struct {
BaseURL string
Client *http.Client
node *codex.CodexNode
config *codex.Config
}
// NewCodexClient creates a new Codex client
func NewCodexClient(host string, port string) *CodexClient {
return &CodexClient{
BaseURL: fmt.Sprintf("http://%s:%s", host, port),
Client: &http.Client{Timeout: 60 * time.Second},
func NewCodexClient(config codex.Config) (*CodexClient, error) {
node, err := codex.New(config)
if err != nil {
return nil, fmt.Errorf("failed to create Codex node: %w", err)
}
return &CodexClient{
node: node,
config: &config,
}, nil
}
func (c CodexClient) Start() error {
return c.node.Start()
}
func (c CodexClient) Stop() error {
return c.node.Stop()
}
func (c CodexClient) Destroy() error {
return c.node.Destroy()
}
// Upload uploads data from a reader to Codex and returns the CID
func (c *CodexClient) Upload(data io.Reader, filename string) (string, error) {
url := fmt.Sprintf("%s/api/codex/v1/data", c.BaseURL)
// Create the HTTP request
req, err := http.NewRequest("POST", url, data)
if err != nil {
return "", fmt.Errorf("failed to create request: %w", err)
}
// Set headers
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Disposition", fmt.Sprintf(`filename="%s"`, filename))
// Send request
resp, err := c.Client.Do(req)
if err != nil {
return "", fmt.Errorf("failed to upload to codex: %w", err)
}
defer resp.Body.Close()
// Check if request was successful
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return "", fmt.Errorf("codex upload failed with status %d: %s", resp.StatusCode, string(body))
}
// Read the CID response
cidBytes, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("failed to read response: %w", err)
}
cid := strings.TrimSpace(string(cidBytes))
return cid, nil
return c.node.UploadReader(codex.UploadOptions{
Filepath: filename,
}, data)
}
// Download downloads data from Codex by CID and writes it to the provided writer
@ -73,201 +58,43 @@ func (c *CodexClient) Download(cid string, output io.Writer) error {
return c.DownloadWithContext(context.Background(), cid, output)
}
func (c *CodexClient) TriggerDownload(cid string) (*CodexManifest, error) {
func (c *CodexClient) TriggerDownload(cid string) (codex.Manifest, error) {
return c.TriggerDownloadWithContext(context.Background(), cid)
}
func (c *CodexClient) HasCid(cid string) (bool, error) {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/exists", c.BaseURL, cid)
resp, err := c.Client.Get(url)
if err != nil {
return false, fmt.Errorf("failed to check cid existence: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return false, fmt.Errorf("cid check failed with status %d: %s", resp.StatusCode, string(body))
}
// Parse JSON response: {"<cid>": <bool>}
var result map[string]bool
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return false, fmt.Errorf("failed to parse response: %w", err)
}
// Validate the CID key matches request
hasCid, exists := result[cid]
if !exists {
return false, fmt.Errorf("response missing CID key %q", cid)
}
return hasCid, nil
err := c.LocalDownload(cid, io.Discard)
return err == nil, nil
}
func (c *CodexClient) RemoveCid(cid string) error {
url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid)
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return fmt.Errorf("failed trying to delete cid: %s, %w", cid, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("cid delete failed with status %d: %s", resp.StatusCode, string(body))
}
return nil
return c.node.Delete(cid)
}
// DownloadWithContext downloads data from Codex by CID with cancellation support
func (c *CodexClient) DownloadWithContext(ctx context.Context, cid string, output io.Writer) error {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/stream", c.BaseURL, cid)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return fmt.Errorf("failed to download from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body))
}
// Use context-aware copy for cancellable streaming
return c.copyWithContext(ctx, output, resp.Body)
return c.node.DownloadStream(cid, codex.DownloadStreamOptions{
Writer: output,
})
}
func (c *CodexClient) LocalDownload(cid string, output io.Writer) error {
return c.LocalDownloadWithContext(context.Background(), cid, output)
return c.node.DownloadStream(cid, codex.DownloadStreamOptions{
Writer: output,
Local: true,
})
}
func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error {
url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return fmt.Errorf("failed to download from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body))
}
// Use context-aware copy for cancellable streaming
return c.copyWithContext(ctx, output, resp.Body)
return c.LocalDownload(cid, output)
}
func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/manifest", c.BaseURL, cid)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch manifest from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("codex fetch manifest failed with status %d: %s", resp.StatusCode, string(body))
}
// Parse JSON response containing manifest
var manifest CodexManifest
if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil {
return nil, fmt.Errorf("failed to parse manifest: %w", err)
}
return &manifest, nil
func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error) {
return c.node.DownloadManifest(cid)
}
func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/network", c.BaseURL, cid)
req, err := http.NewRequestWithContext(ctx, "POST", url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to trigger download from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("codex async download failed with status %d: %s", resp.StatusCode, string(body))
}
// Parse JSON response containing manifest
var manifest CodexManifest
if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil {
return nil, fmt.Errorf("failed to parse download manifest: %w", err)
}
return &manifest, nil
}
// copyWithContext performs io.Copy but respects context cancellation
func (c *CodexClient) copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error {
// Create a buffer for chunked copying
buf := make([]byte, 64*1024) // 64KB buffer
for {
select {
case <-ctx.Done():
return ctx.Err() // Return cancellation error
default:
}
// Read a chunk
n, err := src.Read(buf)
if n > 0 {
// Write the chunk
if _, writeErr := dst.Write(buf[:n]); writeErr != nil {
return fmt.Errorf("failed to write data: %w", writeErr)
}
}
if err == io.EOF {
return nil // Successful completion
}
if err != nil {
return fmt.Errorf("failed to read data: %w", err)
}
}
}
// SetRequestTimeout sets the HTTP client timeout for requests
func (c *CodexClient) SetRequestTimeout(timeout time.Duration) {
c.Client.Timeout = timeout
func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error) {
return c.node.Fetch(cid)
}
// UploadArchive is a convenience method for uploading archive data

63
communities/codex_client_integration_test.go
View File

@ -8,10 +8,10 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"os"
"testing"
"time"
"github.com/codex-storage/codex-go-bindings/codex"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -20,29 +20,22 @@ import (
)
// CodexClientIntegrationTestSuite demonstrates testify's suite functionality for CodexClient integration tests
// These tests exercise real network calls against a running Codex node.
// Required env vars (with defaults):
// - CODEX_HOST (default: localhost)
// - CODEX_API_PORT (default: 8080)
// - CODEX_TIMEOUT_MS (optional; default: 60000)
type CodexClientIntegrationTestSuite struct {
suite.Suite
client *communities.CodexClient
host string
port string
}
// SetupSuite runs once before all tests in the suite
func (suite *CodexClientIntegrationTestSuite) SetupSuite() {
suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost")
suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8080")
suite.client = communities.NewCodexClient(suite.host, suite.port)
// Optional request timeout override
if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" {
if d, err := time.ParseDuration(ms + "ms"); err == nil {
suite.client.SetRequestTimeout(d)
}
var err error
suite.client, err = communities.NewCodexClient(codex.Config{
DataDir: suite.T().TempDir(),
LogFormat: codex.LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
})
if err != nil {
suite.T().Fatalf("Failed to create Codex client: %v", err)
}
}
@ -114,15 +107,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_CheckNonExistingCI
}
func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload() {
// Use port 8001 for this test as specified
client := communities.NewCodexClient(suite.host, "8001")
// Optional request timeout override
if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" {
if d, err := time.ParseDuration(ms + "ms"); err == nil {
client.SetRequestTimeout(d)
}
}
client := communities.NewCodexClientTest(suite.T())
// Generate random payload to ensure proper round-trip verification
payload := make([]byte, 1024)
@ -145,7 +130,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload()
// Trigger async download
manifest, err := client.TriggerDownload(cid)
require.NoError(suite.T(), err, "TriggerDownload failed")
suite.T().Logf("Async download triggered, manifest CID: %s", manifest.CID)
suite.T().Logf("Async download triggered, manifest CID: %s", manifest.Cid)
// Poll HasCid for up to 10 seconds using goroutine and channel
downloadComplete := make(chan bool, 1)
@ -221,27 +206,27 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_FetchManifest() {
manifest, err := suite.client.FetchManifestWithContext(ctx, cid)
require.NoError(suite.T(), err, "FetchManifestWithContext failed")
suite.T().Logf("FetchManifest successful, manifest CID: %s", manifest.CID)
suite.T().Logf("FetchManifest successful, manifest CID: %s", manifest.Cid)
// Verify manifest properties
assert.Equal(suite.T(), cid, manifest.CID, "Manifest CID mismatch")
assert.Equal(suite.T(), cid, manifest.Cid, "Manifest CID mismatch")
// Verify manifest has expected fields
assert.NotEmpty(suite.T(), manifest.Manifest.TreeCid, "Expected TreeCid to be non-empty")
suite.T().Logf("Manifest TreeCid: %s", manifest.Manifest.TreeCid)
assert.NotEmpty(suite.T(), manifest.TreeCid, "Expected TreeCid to be non-empty")
suite.T().Logf("Manifest TreeCid: %s", manifest.TreeCid)
assert.Greater(suite.T(), manifest.Manifest.DatasetSize, int64(0), "Expected DatasetSize > 0")
suite.T().Logf("Manifest DatasetSize: %d", manifest.Manifest.DatasetSize)
assert.Greater(suite.T(), manifest.DatasetSize, 0, "Expected DatasetSize > 0")
suite.T().Logf("Manifest DatasetSize: %d", manifest.DatasetSize)
assert.Greater(suite.T(), manifest.Manifest.BlockSize, 0, "Expected BlockSize > 0")
suite.T().Logf("Manifest BlockSize: %d", manifest.Manifest.BlockSize)
assert.Greater(suite.T(), manifest.BlockSize, 0, "Expected BlockSize > 0")
suite.T().Logf("Manifest BlockSize: %d", manifest.BlockSize)
assert.Equal(suite.T(), "fetch-manifest-test.bin", manifest.Manifest.Filename, "Filename mismatch")
suite.T().Logf("Manifest Filename: %s", manifest.Manifest.Filename)
assert.Equal(suite.T(), "fetch-manifest-test.bin", manifest.Filename, "Filename mismatch")
suite.T().Logf("Manifest Filename: %s", manifest.Filename)
// Log manifest details for verification
suite.T().Logf("Manifest Protected: %v", manifest.Manifest.Protected)
suite.T().Logf("Manifest Mimetype: %s", manifest.Manifest.Mimetype)
suite.T().Logf("Manifest Protected: %v", manifest.Protected)
suite.T().Logf("Manifest Mimetype: %s", manifest.Mimetype)
// Test fetching manifest for non-existent CID (should fail gracefully)
nonExistentCID := "zDvZRwzmNonExistentCID123456789"

12
communities/codex_client_interface.go
View File

@ -3,7 +3,8 @@ package communities
import (
"context"
"io"
"time"
"github.com/codex-storage/codex-go-bindings/codex"
)
// Mock generation instruction above will create a mock in package `mock_communities`
@ -25,16 +26,13 @@ type CodexClientInterface interface {
LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error
// Async download methods
TriggerDownload(cid string) (*CodexManifest, error)
TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error)
TriggerDownload(cid string) (codex.Manifest, error)
TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error)
// Manifest methods
FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error)
FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error)
// CID management methods
HasCid(cid string) (bool, error)
RemoveCid(cid string) error
// Configuration methods
SetRequestTimeout(timeout time.Duration)
}

556
communities/codex_client_test.go
View File

@ -4,10 +4,7 @@ import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
@ -18,24 +15,33 @@ import (
"go-codex-client/communities"
)
func upload(client communities.CodexClient, t *testing.T, buf *bytes.Buffer) string {
filename := "hello.txt"
cid, err := client.Upload(buf, filename)
if err != nil {
t.Fatalf("Failed to upload file: %v", err)
}
if cid == "" {
t.Fatalf("Expected non-empty CID after upload")
}
return cid
}
// CodexClientTestSuite demonstrates testify's suite functionality for CodexClient tests
type CodexClientTestSuite struct {
suite.Suite
client *communities.CodexClient
server *httptest.Server
}
// SetupTest runs before each test method
func (suite *CodexClientTestSuite) SetupTest() {
suite.client = communities.NewCodexClient("localhost", "8080")
suite.client = communities.NewCodexClientTest(suite.T())
}
// TearDownTest runs after each test method
func (suite *CodexClientTestSuite) TearDownTest() {
if suite.server != nil {
suite.server.Close()
suite.server = nil
}
}
// TestCodexClientTestSuite runs the test suite
@ -44,110 +50,33 @@ func TestCodexClientTestSuite(t *testing.T) {
}
func (suite *CodexClientTestSuite) TestUpload_Success() {
// Arrange a fake Codex server that validates headers and returns a CID
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if r.URL.Path != "/api/codex/v1/data" {
w.WriteHeader(http.StatusNotFound)
return
}
if ct := r.Header.Get("Content-Type"); ct != "application/octet-stream" {
w.WriteHeader(http.StatusBadRequest)
return
}
if cd := r.Header.Get("Content-Disposition"); cd != "filename=\"hello.txt\"" {
w.WriteHeader(http.StatusBadRequest)
return
}
_, _ = io.ReadAll(r.Body) // consume body
_ = r.Body.Close()
w.WriteHeader(http.StatusOK)
// Codex returns CIDv1 base58btc
// prefix: zDv
// - z = multibase prefix for base58btc
// - Dv = CIDv1 prefix for raw codex
// we add a newline to simulate real response
_, _ = w.Write([]byte("zDvZRwzmTestCID123\n"))
}))
suite.client.BaseURL = suite.server.URL
// Act
cid, err := suite.client.Upload(bytes.NewReader([]byte("payload")), "hello.txt")
// Assert
require.NoError(suite.T(), err)
// Codex uses CIDv1 with base58btc encoding (prefix: zDv)
assert.Equal(suite.T(), "zDvZRwzmTestCID123", cid)
}
func (suite *CodexClientTestSuite) TestDownload_Success() {
const wantCID = "zDvZRwzm"
const payload = "hello from codex"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if r.URL.Path != "/api/codex/v1/data/"+wantCID+"/network/stream" {
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(payload))
}))
suite.client.BaseURL = suite.server.URL
var buf bytes.Buffer
err := suite.client.Download(wantCID, &buf)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), payload, buf.String())
assert.Equal(suite.T(), "zDvZRwzmBEaJ338xaCHbKbGAJ4X41YyccS6eyorrYBbmPnWuLxCh", cid)
}
func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() {
const cid = "zDvZRwzm"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/codex/v1/data/"+cid+"/network/stream" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
flusher, _ := w.(http.Flusher)
w.WriteHeader(http.StatusOK)
// Stream data slowly so the request can be canceled
for i := 0; i < 1000; i++ {
select {
case <-r.Context().Done():
return
default:
}
if _, err := w.Write([]byte("x")); err != nil {
// Client likely went away; stop writing
return
}
if flusher != nil {
flusher.Flush()
}
time.Sleep(10 * time.Millisecond)
}
}))
suite.client.BaseURL = suite.server.URL
// skip test
suite.T().Skip("Wait for cancellation support PR to be merged in codex-go-bindings")
len := 1024 * 1024 * 50
buf := bytes.NewBuffer(make([]byte, len))
cid := upload(*suite.client, suite.T(), buf)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
err := suite.client.DownloadWithContext(ctx, cid, io.Discard)
channelError := make(chan error, 1)
go func() {
err := suite.client.DownloadWithContext(ctx, cid, io.Discard)
channelError <- err
}()
cancel()
err := <-channelError
require.Error(suite.T(), err)
// Accept either canceled or deadline exceeded depending on timing
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
@ -160,31 +89,20 @@ func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() {
}
func (suite *CodexClientTestSuite) TestHasCid_Success() {
const payload = "hello from codex"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
tests := []struct {
name string
cid string
hasIt bool
wantBool bool
}{
{"has CID returns true", "zDvZRwzmTestCID", true, true},
{"has CID returns false", "zDvZRwzmTestCID", false, false},
{"has CID returns true", cid, true},
{"has CID returns false", "zDvZRwzmBEaJ338xaCHbKbGAJ4X41YyccS6eyorrYBbmPnWuLxCe", false},
}
for _, tt := range tests {
suite.Run(tt.name, func() {
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/codex/v1/data/"+tt.cid+"/exists" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
// Return JSON: {"<cid>": <bool>}
fmt.Fprintf(w, `{"%s": %t}`, tt.cid, tt.hasIt)
}))
suite.client.BaseURL = suite.server.URL
got, err := suite.client.HasCid(tt.cid)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), tt.wantBool, got, "HasCid(%q) = %v, want %v", tt.cid, got, tt.wantBool)
@ -192,181 +110,42 @@ func (suite *CodexClientTestSuite) TestHasCid_Success() {
}
}
func (suite *CodexClientTestSuite) TestHasCid_RequestError() {
// Create a server and immediately close it to trigger connection error
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
suite.server.Close() // Close immediately so connection fails
func (suite *CodexClientTestSuite) TestDownload_Success() {
const payload = "hello from codex"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
suite.client.BaseURL = suite.server.URL // Use the closed server's URL
got, err := suite.client.HasCid("zDvZRwzmTestCID")
require.Error(suite.T(), err)
assert.False(suite.T(), got, "expected false on error")
}
func (suite *CodexClientTestSuite) TestHasCid_CidMismatch() {
const requestCid = "zDvZRwzmRequestCID"
const responseCid = "zDvZRwzmDifferentCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
// Return a different CID in the response
fmt.Fprintf(w, `{"%s": true}`, responseCid)
}))
suite.client.BaseURL = suite.server.URL
got, err := suite.client.HasCid(requestCid)
require.Error(suite.T(), err, "expected error for CID mismatch")
assert.False(suite.T(), got, "expected false on CID mismatch")
// Check error message mentions the missing/mismatched CID
assert.Contains(suite.T(), err.Error(), requestCid, "error should mention request CID")
var buf bytes.Buffer
err := suite.client.Download(cid, &buf)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), payload, buf.String())
}
func (suite *CodexClientTestSuite) TestRemoveCid_Success() {
const testCid = "zDvZRwzmTestCID"
const payload = "hello from codex"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if r.URL.Path != "/api/codex/v1/data/"+testCid {
w.WriteHeader(http.StatusNotFound)
return
}
// DELETE should return 204 No Content
w.WriteHeader(http.StatusNoContent)
}))
suite.client.BaseURL = suite.server.URL
err := suite.client.RemoveCid(testCid)
err := suite.client.RemoveCid(cid)
require.NoError(suite.T(), err)
}
func (suite *CodexClientTestSuite) TestRemoveCid_Error() {
const testCid = "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Return error status
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("server error"))
}))
suite.client.BaseURL = suite.server.URL
err := suite.client.RemoveCid(testCid)
require.Error(suite.T(), err)
assert.Contains(suite.T(), err.Error(), "500", "error should mention status 500")
}
func (suite *CodexClientTestSuite) TestTriggerDownload() {
const testCid = "zDvZRwzmTestCID"
const expectedManifest = `{
"cid": "zDvZRwzmTestCID",
"manifest": {
"treeCid": "zDvZRwzmTreeCID",
"datasetSize": 1024,
"blockSize": 65536,
"protected": false,
"filename": "test-file.bin",
"mimetype": "application/octet-stream"
}
}`
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if r.URL.Path != "/api/codex/v1/data/"+testCid+"/network" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(expectedManifest))
}))
suite.client.BaseURL = suite.server.URL
const payload = "hello from codex"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
ctx := context.Background()
manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid)
manifest, err := suite.client.TriggerDownloadWithContext(ctx, cid)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), testCid, manifest.CID)
assert.Equal(suite.T(), "zDvZRwzmTreeCID", manifest.Manifest.TreeCid)
assert.Equal(suite.T(), int64(1024), manifest.Manifest.DatasetSize)
assert.Equal(suite.T(), "test-file.bin", manifest.Manifest.Filename)
}
func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_RequestError() {
// Create a server and immediately close it to trigger connection error
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
suite.server.Close()
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.TriggerDownloadWithContext(ctx, "zDvZRwzmRigWseNB7WqmudkKAPgZmrDCE9u5cY4KvCqhRo9Ki")
require.Error(suite.T(), err)
assert.Nil(suite.T(), manifest, "expected nil manifest on error")
}
func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_JSONParseError() {
const testCid = "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
// Return invalid JSON
w.Write([]byte(`{"invalid": json}`))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid)
require.Error(suite.T(), err, "expected JSON parse error")
assert.Nil(suite.T(), manifest, "expected nil manifest on parse error")
assert.Contains(suite.T(), err.Error(), "failed to parse download manifest", "error should mention parse failure")
}
func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_HTTPError() {
const testCid = "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("CID not found"))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid)
require.Error(suite.T(), err, "expected error for 404 status")
assert.Nil(suite.T(), manifest, "expected nil manifest on HTTP error")
assert.Contains(suite.T(), err.Error(), "404", "error should mention status 404")
assert.Equal(suite.T(), cid, manifest.Cid)
assert.Equal(suite.T(), "zDzSvJTf7mGkC3yuiVGco7Qc6s4LA8edye9inT4w2QqHnfbuRvMr", manifest.TreeCid)
assert.Equal(suite.T(), len(payload), manifest.DatasetSize)
assert.Equal(suite.T(), "hello.txt", manifest.Filename)
}
func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation() {
suite.T().Skip("Not sure if we are going to have cancellation in trigger download")
const testCid = "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Simulate slow response to allow cancellation
select {
case <-r.Context().Done():
return
case <-time.After(200 * time.Millisecond):
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"cid": "test"}`))
}
}))
suite.client.BaseURL = suite.server.URL
// Cancel after 50ms (before server responds)
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
@ -385,99 +164,43 @@ func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation()
}
func (suite *CodexClientTestSuite) TestLocalDownload() {
testData := []byte("test data for local download")
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Verify request method and path
assert.Equal(suite.T(), "GET", r.Method, "Expected GET request")
expectedPath := "/api/codex/v1/data/" + testCid
assert.Equal(suite.T(), expectedPath, r.URL.Path, "Expected correct path")
w.WriteHeader(http.StatusOK)
w.Write(testData)
}))
suite.client.BaseURL = suite.server.URL
const payload = "test data for local download"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
var buf bytes.Buffer
err := suite.client.LocalDownload(testCid, &buf)
err := suite.client.LocalDownload(cid, &buf)
require.NoError(suite.T(), err, "LocalDownload failed")
assert.Equal(suite.T(), testData, buf.Bytes(), "Downloaded data mismatch")
assert.Equal(suite.T(), payload, buf.String(), "Downloaded data mismatch")
}
func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Success() {
testData := []byte("test data for local download with context")
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Verify request method and path
assert.Equal(suite.T(), "GET", r.Method, "Expected GET request")
expectedPath := "/api/codex/v1/data/" + testCid
assert.Equal(suite.T(), expectedPath, r.URL.Path, "Expected correct path")
w.WriteHeader(http.StatusOK)
w.Write(testData)
}))
suite.client.BaseURL = suite.server.URL
const payload = "test data for local download with context"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
ctx := context.Background()
var buf bytes.Buffer
err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf)
err := suite.client.LocalDownloadWithContext(ctx, cid, &buf)
require.NoError(suite.T(), err, "LocalDownloadWithContext failed")
assert.Equal(suite.T(), testData, buf.Bytes(), "Downloaded data mismatch")
}
func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_RequestError() {
// Create a server and immediately close it to trigger connection error
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
suite.server.Close()
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
var buf bytes.Buffer
err := suite.client.LocalDownloadWithContext(ctx, "zDvZRwzmTestCID", &buf)
require.Error(suite.T(), err, "Expected error due to closed server")
assert.Contains(suite.T(), err.Error(), "failed to download from codex")
}
func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_HTTPError() {
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("CID not found in local storage"))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
var buf bytes.Buffer
err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf)
require.Error(suite.T(), err, "Expected error for HTTP 404")
assert.Contains(suite.T(), err.Error(), "404", "Expected '404' in error message")
assert.Equal(suite.T(), payload, buf.String(), "Downloaded data mismatch")
}
func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() {
testCid := "zDvZRwzmTestCID"
suite.T().Skip("Wait for cancellation support PR to be merged in codex-go-bindings")
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Simulate a slow response
time.Sleep(100 * time.Millisecond)
w.WriteHeader(http.StatusOK)
w.Write([]byte("slow response"))
}))
suite.client.BaseURL = suite.server.URL
// Create a context with a very short timeout
len := 1024 * 1024 * 50
buf := bytes.NewBuffer(make([]byte, len))
cid := upload(*suite.client, suite.T(), buf)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel()
var buf bytes.Buffer
err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf)
channelError := make(chan error, 1)
go func() {
err := suite.client.LocalDownloadWithContext(ctx, cid, io.Discard)
channelError <- err
}()
cancel()
err := <-channelError
require.Error(suite.T(), err, "Expected context cancellation error")
// Accept either canceled or deadline exceeded depending on timing
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
@ -490,107 +213,28 @@ func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() {
}
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Success() {
testCid := "zDvZRwzmTestCID"
expectedManifest := `{
"cid": "zDvZRwzmTestCID",
"manifest": {
"treeCid": "zDvZRwzmTreeCID123",
"datasetSize": 1024,
"blockSize": 256,
"protected": true,
"filename": "test-file.bin",
"mimetype": "application/octet-stream"
}
}`
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(suite.T(), http.MethodGet, r.Method)
expectedPath := fmt.Sprintf("/api/codex/v1/data/%s/network/manifest", testCid)
assert.Equal(suite.T(), expectedPath, r.URL.Path)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(expectedManifest))
}))
suite.client.BaseURL = suite.server.URL
const payload = "hello from codex"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
ctx := context.Background()
manifest, err := suite.client.FetchManifestWithContext(ctx, testCid)
manifest, err := suite.client.FetchManifestWithContext(ctx, cid)
require.NoError(suite.T(), err, "Expected no error")
require.NotNil(suite.T(), manifest, "Expected manifest, got nil")
assert.Equal(suite.T(), testCid, manifest.CID)
assert.Equal(suite.T(), "zDvZRwzmTreeCID123", manifest.Manifest.TreeCid)
assert.Equal(suite.T(), int64(1024), manifest.Manifest.DatasetSize)
assert.Equal(suite.T(), 256, manifest.Manifest.BlockSize)
assert.True(suite.T(), manifest.Manifest.Protected, "Expected Protected to be true")
assert.Equal(suite.T(), "test-file.bin", manifest.Manifest.Filename)
assert.Equal(suite.T(), "application/octet-stream", manifest.Manifest.Mimetype)
}
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_RequestError() {
// Create a server and immediately close it to trigger connection error
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
suite.server.Close()
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.FetchManifestWithContext(ctx, "test-cid")
require.Error(suite.T(), err, "Expected error for closed server")
assert.Nil(suite.T(), manifest, "Expected nil manifest on error")
assert.Contains(suite.T(), err.Error(), "failed to fetch manifest from codex")
}
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_HTTPError() {
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("Manifest not found"))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.FetchManifestWithContext(ctx, testCid)
require.Error(suite.T(), err, "Expected error for HTTP 404")
assert.Nil(suite.T(), manifest, "Expected nil manifest on error")
assert.Contains(suite.T(), err.Error(), "404", "Expected '404' in error message")
}
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_JSONParseError() {
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte("invalid json {"))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.FetchManifestWithContext(ctx, testCid)
require.Error(suite.T(), err, "Expected error for invalid JSON")
assert.Nil(suite.T(), manifest, "Expected nil manifest on JSON parse error")
assert.Contains(suite.T(), err.Error(), "failed to parse manifest", "Expected 'failed to parse manifest' in error message")
assert.Equal(suite.T(), cid, manifest.Cid)
assert.Equal(suite.T(), "zDzSvJTf7mGkC3yuiVGco7Qc6s4LA8edye9inT4w2QqHnfbuRvMr", manifest.TreeCid)
assert.Equal(suite.T(), len(payload), manifest.DatasetSize)
assert.Equal(suite.T(), 65536, manifest.BlockSize)
assert.True(suite.T(), !manifest.Protected, "Expected Protected to be false")
assert.Equal(suite.T(), "hello.txt", manifest.Filename)
assert.Equal(suite.T(), "text/plain", manifest.Mimetype)
}
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() {
suite.T().Skip("Not sure if we are going to have cancellation in fetch manifest")
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Simulate a slow response
time.Sleep(100 * time.Millisecond)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"cid": "test"}`))
}))
suite.client.BaseURL = suite.server.URL
// Create a context with a very short timeout
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel()
@ -607,4 +251,30 @@ func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() {
suite.T().Fatalf("expected context cancellation, got: %v", err)
}
}
buf := bytes.NewBuffer([]byte("Hello World!"))
if buf.Len() != manifest.DatasetSize {
suite.T().Errorf("expected size %d, got %d", buf.Len(), manifest.DatasetSize)
}
defaultBlockSize := 1024 * 64
if manifest.BlockSize != defaultBlockSize {
suite.T().Errorf("expected block size %d, got %d", defaultBlockSize, manifest.BlockSize)
}
if manifest.Filename != "test.txt" {
suite.T().Errorf("expected filename %q, got %q", "test.txt", manifest.Filename)
}
if manifest.Protected {
suite.T().Errorf("expected protected to be false, got true")
}
if manifest.Mimetype != "text/plain" {
suite.T().Errorf("expected mimetype %q, got %q", "text/plain", manifest.Mimetype)
}
if manifest.TreeCid == "" {
suite.T().Errorf("expected non-empty TreeCid")
}
}

8
communities/codex_index_downloader.go
View File

@ -79,19 +79,19 @@ func (d *CodexIndexDownloader) GotManifest() <-chan struct{} {
}
// Verify that the CID matches our configured indexCid
if manifest.CID != d.indexCid {
if manifest.Cid != d.indexCid {
d.mu.Lock()
d.downloadError = fmt.Errorf("manifest CID mismatch: expected %s, got %s", d.indexCid, manifest.CID)
d.downloadError = fmt.Errorf("manifest CID mismatch: expected %s, got %s", d.indexCid, manifest.Cid)
d.mu.Unlock()
d.logger.Debug("manifest CID mismatch",
zap.String("expected", d.indexCid),
zap.String("got", manifest.CID))
zap.String("got", manifest.Cid))
return
}
// Store the dataset size for later use - this indicates success
d.mu.Lock()
d.datasetSize = manifest.Manifest.DatasetSize
d.datasetSize = int64(manifest.DatasetSize)
d.mu.Unlock()
// Success! Close the channel to signal completion

13
communities/codex_index_downloader_integration_test.go
View File

@ -30,23 +30,12 @@ type CodexIndexDownloaderIntegrationTestSuite struct {
suite.Suite
client *communities.CodexClient
testDir string
host string
port string
logger *zap.Logger
}
// SetupSuite runs once before all tests in the suite
func (suite *CodexIndexDownloaderIntegrationTestSuite) SetupSuite() {
suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost")
suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8001")
suite.client = communities.NewCodexClient(suite.host, suite.port)
// Optional request timeout override
if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" {
if d, err := time.ParseDuration(ms + "ms"); err == nil {
suite.client.SetRequestTimeout(d)
}
}
suite.client = communities.NewCodexClientTest(suite.T())
// Create logger
suite.logger, _ = zap.NewDevelopment()

43
communities/codex_index_downloader_test.go
View File

@ -9,6 +9,7 @@ import (
"testing"
"time"
"github.com/codex-storage/codex-go-bindings/codex"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
@ -79,12 +80,12 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_SuccessClosesChannel
filePath := filepath.Join(suite.testDir, "index.bin")
// Setup mock to return a successful manifest
expectedManifest := &communities.CodexManifest{
CID: testCid,
expectedManifest := codex.Manifest{
Cid: testCid,
}
expectedManifest.Manifest.DatasetSize = 1024
expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID"
expectedManifest.Manifest.BlockSize = 65536
expectedManifest.DatasetSize = 1024
expectedManifest.TreeCid = "zDvZRwzmTreeCID"
expectedManifest.BlockSize = 65536
suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid).
@ -119,7 +120,7 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_ErrorDoesNotCloseCha
// Setup mock to return an error
suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid).
Return(nil, errors.New("fetch error"))
Return(codex.Manifest{}, errors.New("fetch error"))
// Create downloader
downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
@ -154,10 +155,10 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_CidMismatchDoesNotCl
filePath := filepath.Join(suite.testDir, "index.bin")
// Setup mock to return a manifest with different CID
mismatchedManifest := &communities.CodexManifest{
CID: differentCid, // Different CID!
mismatchedManifest := codex.Manifest{
Cid: differentCid, // Different CID!
}
mismatchedManifest.Manifest.DatasetSize = 1024
mismatchedManifest.DatasetSize = 1024
suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid).
@ -198,12 +199,12 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_Cancellation() {
fetchCalled := make(chan struct{})
suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid).
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) {
DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
close(fetchCalled) // Signal that fetch was called
// Wait for context cancellation
<-ctx.Done()
return nil, ctx.Err()
return codex.Manifest{}, ctx.Err()
})
// Create downloader
@ -250,11 +251,11 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize()
expectedSize := int64(2048)
// Setup mock to return a manifest with specific dataset size
expectedManifest := &communities.CodexManifest{
CID: testCid,
expectedManifest := codex.Manifest{
Cid: testCid,
}
expectedManifest.Manifest.DatasetSize = expectedSize
expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID"
expectedManifest.DatasetSize = int(expectedSize)
expectedManifest.TreeCid = "zDvZRwzmTreeCID"
suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid).
@ -278,7 +279,7 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize()
}
// Verify dataset size was recorded correctly
assert.Equal(suite.T(), expectedSize, downloader.GetDatasetSize(), "Dataset size should match manifest")
assert.Equal(suite.T(), int64(expectedSize), downloader.GetDatasetSize(), "Dataset size should match manifest")
suite.T().Logf("✅ Dataset size correctly recorded: %d", downloader.GetDatasetSize())
// Verify no error was recorded
@ -500,13 +501,13 @@ func (suite *CodexIndexDownloaderTestSuite) TestDownloadIndexFile_ErrorHandling(
func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() {
testCid := "zDvZRwzmTestCID123"
filePath := filepath.Join(suite.testDir, "index.bin")
expectedSize := int64(4096)
expectedSize := 4096
// Setup mock to return a manifest
expectedManifest := &communities.CodexManifest{
CID: testCid,
expectedManifest := codex.Manifest{
Cid: testCid,
}
expectedManifest.Manifest.DatasetSize = expectedSize
expectedManifest.DatasetSize = expectedSize
suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid).
@ -523,6 +524,6 @@ func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() {
<-manifestChan
// Now Length should return the dataset size
assert.Equal(suite.T(), expectedSize, downloader.Length(), "Length should return dataset size")
assert.Equal(suite.T(), int64(expectedSize), downloader.Length(), "Length should return dataset size")
suite.T().Logf("✅ Length() correctly returns dataset size: %d", downloader.Length())
}

14
communities/codex_manifest.go
View File

@ -1,14 +0,0 @@
package communities
// CodexManifest represents the manifest structure returned by Codex API
type CodexManifest struct {
CID string `json:"cid"`
Manifest struct {
TreeCid string `json:"treeCid"`
DatasetSize int64 `json:"datasetSize"`
BlockSize int `json:"blockSize"`
Protected bool `json:"protected"`
Filename string `json:"filename"`
Mimetype string `json:"mimetype"`
} `json:"manifest"`
}

14
communities/mock/codex_client_interface.go
View File

@ -11,11 +11,11 @@ package mock_communities
import (
context "context"
communities "go-codex-client/communities"
io "io"
reflect "reflect"
time "time"
"github.com/codex-storage/codex-go-bindings/codex"
gomock "go.uber.org/mock/gomock"
)
@ -72,10 +72,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) DownloadWithContext(ctx, cid, ou
}
// FetchManifestWithContext mocks base method.
func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) {
func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "FetchManifestWithContext", ctx, cid)
ret0, _ := ret[0].(*communities.CodexManifest)
ret0, _ := ret[0].(codex.Manifest)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -156,10 +156,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) SetRequestTimeout(timeout any) *
}
// TriggerDownload mocks base method.
func (m *MockCodexClientInterface) TriggerDownload(cid string) (*communities.CodexManifest, error) {
func (m *MockCodexClientInterface) TriggerDownload(cid string) (codex.Manifest, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "TriggerDownload", cid)
ret0, _ := ret[0].(*communities.CodexManifest)
ret0, _ := ret[0].(codex.Manifest)
ret1, _ := ret[1].(error)
return ret0, ret1
}
@ -171,10 +171,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) TriggerDownload(cid any) *gomock
}
// TriggerDownloadWithContext mocks base method.
func (m *MockCodexClientInterface) TriggerDownloadWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) {
func (m *MockCodexClientInterface) TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "TriggerDownloadWithContext", ctx, cid)
ret0, _ := ret[0].(*communities.CodexManifest)
ret0, _ := ret[0].(codex.Manifest)
ret1, _ := ret[1].(error)
return ret0, ret1
}

37
communities/testutil.go Normal file
View File

@ -0,0 +1,37 @@
package communities
import (
"testing"
"github.com/codex-storage/codex-go-bindings/codex"
)
func NewCodexClientTest(t *testing.T) *CodexClient {
client, err := NewCodexClient(codex.Config{
DataDir: t.TempDir(),
LogFormat: codex.LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
DiscoveryPort: 8092,
})
if err != nil {
t.Fatalf("Failed to create Codex node: %v", err)
}
err = client.Start()
if err != nil {
t.Fatalf("Failed to start Codex node: %v", err)
}
t.Cleanup(func() {
if err := client.Stop(); err != nil {
t.Logf("cleanup codex: %v", err)
}
if err := client.Destroy(); err != nil {
t.Logf("cleanup codex: %v", err)
}
})
return client
}

3
go.mod
View File

@ -1,6 +1,6 @@
module go-codex-client
go 1.23.0
go 1.24.0
require (
github.com/stretchr/testify v1.11.1
@ -10,6 +10,7 @@ require (
)
require (
github.com/codex-storage/codex-go-bindings v0.0.22 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.uber.org/multierr v1.10.0 // indirect

2
go.sum
View File

@ -1,3 +1,5 @@
github.com/codex-storage/codex-go-bindings v0.0.22 h1:53nOqLzgfvR3KdghFAKDoREoW+n12ewvNf8Zf3Pdobc=
github.com/codex-storage/codex-go-bindings v0.0.22/go.mod h1:hP/n9iDZqQP4MytkgUepl3yMMsZy5Jbk9lQbbbVJ51Q=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=