logos-storage/logos-storage-go
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-go.git synced 2026-01-04 06:13:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add codex library integration

Browse Source
This commit is contained in:
Arnaud 2025-10-28 06:56:13 +01:00
parent 0c4d537303
commit a74ac84124
No known key found for this signature in database
GPG Key ID: 20E40A5D3110766F
21 changed files with 424 additions and 938 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -21,3 +21,5 @@ coverage*.cov
# Logs # Logs
*.log *.log
libs

11
.vscode/settings.json vendored
View File

@ -1,6 +1,13 @@
{ {
"go.testTags": "codex_integration", "go.testTags": "codex_integration",
"gopls": { "gopls": {
"buildFlags": ["-tags=codex_integration"] "buildFlags": [
"-tags=integration"
]
},
"go.toolsEnvVars": {
"CGO_ENABLED": "1",
"CGO_CFLAGS": "-I${workspaceFolder}/libs",
"CGO_LDFLAGS": "-L${workspaceFolder}/libs -lcodex -Wl,-rpath,${workspaceFolder}/libs"
} }
} }

46
Makefile Normal file
View File

@ -0,0 +1,46 @@
# Destination folder for the downloaded libraries
LIBS_DIR := $(abspath ./libs)
# Flags for CGO to find the headers and the shared library
UNAME_S := $(shell uname -s)
CGO_CFLAGS := -I$(LIBS_DIR)
CGO_LDFLAGS := -L$(LIBS_DIR) -lcodex -Wl,-rpath,$(LIBS_DIR)
ifeq ($(OS),Windows_NT)
BIN_NAME := codex-go.exe
else
BIN_NAME := codex-go
endif
# Configuration for fetching the right binary
OS ?= "linux"
ARCH ?= "amd64"
VERSION ?= "v0.0.22"
DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(VERSION)/codex-${OS}-${ARCH}.zip"
fetch:
@echo "Fetching libcodex from GitHub Actions from: ${DOWNLOAD_URL}"
curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${OS}-${ARCH}.zip ${DOWNLOAD_URL}
unzip -o -qq $(LIBS_DIR)/codex-${OS}-${ARCH}.zip -d $(LIBS_DIR)
rm -f $(LIBS_DIR)/*.zip
build:
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o $(BIN_NAME) main.go
build-upload:
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-upload ./cmd/upload
build-download:
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-download ./cmd/download
test:
@echo "Running unit tests..."
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -v ./communities
test-integration:
@echo "Running tests..."
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -v -tags=codex_integration ./communities -run Integration -timeout 15s
clean:
rm -f $(BIN_NAME)
rm -Rf $(LIBS_DIR)/*

147
README.md
View File

@ -10,25 +10,12 @@ A lightweight Go client utility for interacting with Codex client.
We will be running codex client, and then use a small testing utility to check if the low level abstraction - CodexClient - correctly uploads and downloads the content. We will be running codex client, and then use a small testing utility to check if the low level abstraction - CodexClient - correctly uploads and downloads the content.
### Running CodexClient ### Integration Codex library
I often remove some logging noise, by slightly changing the build You need to download the library file by using:
params in `build.nims` (nim-codex):
```nim ```sh
task codex, "build codex binary": make fetch
buildBinary "codex",
# params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
params =
"-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE"
```
You see a slightly more selective `params` in the `codex` task.
To run the client I use the following command:
```bash
./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE
``` ```
### Building codex-upload and codex-download utilities ### Building codex-upload and codex-download utilities
@ -36,8 +23,8 @@ To run the client I use the following command:
Use the following command to build the `codex-upload` and `codex-download` utilities: Use the following command to build the `codex-upload` and `codex-download` utilities:
```bash ```bash
go build -o bin/codex-upload ./cmd/upload make build-upload
go build -o bin/codex-download ./cmd/download make build-download
``` ```
### Uploading content to Codex ### Uploading content to Codex
@ -45,8 +32,8 @@ Now, using the `codex-upload` utility, we can upload the content to Codex as fol
```bash ```bash
~/code/local/go-codex-client ~/code/local/go-codex-client
❯ ./bin/codex-upload -file test-data.bin -host localhost -port 8001 ❯ ./bin/codex-upload -file test-data.bin
Uploading test-data.bin (43 bytes) to Codex at localhost:8001... Uploading test-data.bin (43 bytes) to Codex
✅ Upload successful! ✅ Upload successful!
CID: zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V CID: zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V
``` ```
@ -57,8 +44,8 @@ Now, having the content uploaded to Codex - let's get it back using the `codex-d
```bash ```bash
~/code/local/go-codex-client ~/code/local/go-codex-client
❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin -host localhost -port 8001 ❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin
Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex at localhost:8001... Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex...
✅ Download successful! ✅ Download successful!
Saved to: output.bin Saved to: output.bin
``` ```
@ -85,115 +72,23 @@ next section.
To run all unit tests: To run all unit tests:
```bash ```bash
❯ go test -v ./communities -count 1 ❯ make test
=== RUN TestUpload_Success
--- PASS: TestUpload_Success (0.00s)
=== RUN TestDownload_Success
--- PASS: TestDownload_Success (0.00s)
=== RUN TestDownloadWithContext_Cancel
--- PASS: TestDownloadWithContext_Cancel (0.04s)
PASS
ok go-codex-client/communities 0.044s
``` ```
To be more selective, e.g. in order to run all the tests from To run the integration test, use `test-integration`:
`CodexArchiveDownloaderSuite`, run:
```bash ```bash
go test -v ./communities -run CodexArchiveDownloader -count 1 make test-integration
``` ```
or for an individual test from that suite:
```bash
go test -v ./communities -run TestCodexArchiveDownloaderSuite/TestCancellationDuringPolling -count 1
```
You can also use `gotestsum` to run the tests (you may need to install it first, e.g. `go install gotest.tools/gotestsum@v1.13.0`):
```bash
gotestsum --packages="./communities" -f testname --rerun-fails -- -count 1
```
For a more verbose output including logs use `-f standard-verbose`, e.g.:
```bash
gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -v -count 1
```
To be more selective, e.g. in order to run all the tests from
`CodexArchiveDownloaderSuite`, run:
```bash
gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArchiveDownloader -count 1
```
or for an individual test from that suite:
```bash
gotestsum --packages="./communities" -f testname --rerun-fails -- -run TestCodexArchiveDownloaderSuite/TestCancellationDuringPolling -count 1
```
Notice, that the `-run` flag accepts a regular expression that matches against the full test path, so you can be more concise in naming if necessary, e.g.:
```bash
gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArchiveDownloader/Cancellation -count 1
```
This also applies to native `go test` command.
### Running integration tests
When building Codex client for testing like here, I often remove some logging noise, by slightly changing the build params in `build.nims`:
```nim
task codex, "build codex binary":
buildBinary "codex",
# params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE"
params =
"-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE"
```
You see a slightly more selective `params` in the `codex` task.
To start Codex client, use e.g.:
```bash
./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE
```
To run the integration test, use `codex_integration` tag and narrow the scope using `-run Integration`:
```bash
CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s
```
This will run all integration tests, including CodexClient integration tests.
To make sure that the test is actually run and not cached, use `count` option:
```bash
CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s -count 1
```
To be more specific and only run the tests related to, e.g. index downloader or archive
downloader you can use:
```bash
CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexIndexDownloaderIntegration -timeout 15s -count 1
CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexArchiveDownloaderIntegration -timeout 15s -count 1
```
and then, if you prefer to use `gotestsum`:
```bash
CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexIndexDownloaderIntegration -v -count 1
CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexArchiveDownloaderIntegration -v -count 1
```
or to run all integration tests (including CodexClient integration tests):
```bash
CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -v -count 1 -run Integration
```
I prefer to be more selective when running integration tests.
### Regenerating artifacts ### Regenerating artifacts
Everything you need comes included in the repo. But if you decide to change things, Everything you need comes included in the repo. But if you decide to change things,

29
cmd/download/main.go
View File

@ -5,14 +5,15 @@ import (
"fmt" "fmt"
"log" "log"
"os" "os"
"path"
"go-codex-client/communities" // Import the local communities package "go-codex-client/communities" // Import the local communities package
"github.com/codex-storage/codex-go-bindings/codex"
) )
func main() { func main() {
var ( var (
host = flag.String("host", "localhost", "Codex host")
port = flag.String("port", "8080", "Codex port")
cid = flag.String("cid", "", "CID of the file to download") cid = flag.String("cid", "", "CID of the file to download")
file = flag.String("file", "downloaded-file.bin", "File to save the downloaded data") file = flag.String("file", "downloaded-file.bin", "File to save the downloaded data")
) )
@ -24,7 +25,20 @@ func main() {
} }
// Create Codex client // Create Codex client
client := communities.NewCodexClient(*host, *port) client, err := communities.NewCodexClient(codex.Config{
LogFormat: codex.LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
LogLevel: "ERROR",
DataDir: path.Join(os.TempDir(), "codex-client-data"),
})
if err != nil {
log.Fatalf("Failed to create CodexClient: %v", err)
}
if err := client.Start(); err != nil {
log.Fatalf("Failed to start CodexClient: %v", err)
}
// Create output file // Create output file
outputFile, err := os.Create(*file) outputFile, err := os.Create(*file)
@ -33,8 +47,6 @@ func main() {
} }
defer outputFile.Close() defer outputFile.Close()
fmt.Printf("Downloading CID %s from Codex at %s:%s...\n", *cid, *host, *port)
// Download data - pass the io.Writer (outputFile), not the string // Download data - pass the io.Writer (outputFile), not the string
err = client.Download(*cid, outputFile) err = client.Download(*cid, outputFile)
if err != nil { if err != nil {
@ -43,6 +55,13 @@ func main() {
log.Fatalf("Download failed: %v", err) log.Fatalf("Download failed: %v", err)
} }
if err := client.Stop(); err != nil {
log.Printf("Warning: Failed to stop CodexClient: %v", err)
}
if err := client.Destroy(); err != nil {
log.Printf("Warning: Failed to stop CodexClient: %v", err)
}
fmt.Printf("✅ Download successful!\n") fmt.Printf("✅ Download successful!\n")
fmt.Printf("Saved to: %s\n", *file) fmt.Printf("Saved to: %s\n", *file)
} }

29
cmd/upload/main.go
View File

@ -6,14 +6,15 @@ import (
"fmt" "fmt"
"log" "log"
"os" "os"
"path"
"go-codex-client/communities" // Import the local communities package "go-codex-client/communities" // Import the local communities package
"github.com/codex-storage/codex-go-bindings/codex"
) )
func main() { func main() {
var ( var (
host = flag.String("host", "localhost", "Codex host")
port = flag.String("port", "8080", "Codex port")
file = flag.String("file", "test-data.bin", "File to upload") file = flag.String("file", "test-data.bin", "File to upload")
filename = flag.String("name", "", "Filename to use in upload (defaults to actual filename)") filename = flag.String("name", "", "Filename to use in upload (defaults to actual filename)")
) )
@ -31,15 +32,35 @@ func main() {
uploadName = *file uploadName = *file
} }
fmt.Printf("Uploading %s (%d bytes) to Codex at %s:%s...\n", *file, len(data), *host, *port) fmt.Printf("Uploading %s (%d bytes) to Codex...\n", *file, len(data))
// Create Codex client and upload // Create Codex client and upload
client := communities.NewCodexClient(*host, *port) client, err := communities.NewCodexClient(codex.Config{
LogFormat: codex.LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
LogLevel: "ERROR",
DataDir: path.Join(os.TempDir(), "codex-client-data"),
})
if err != nil {
log.Fatalf("Failed to create CodexClient: %v", err)
}
if err := client.Start(); err != nil {
log.Fatalf("Failed to start CodexClient: %v", err)
}
cid, err := client.Upload(bytes.NewReader(data), uploadName) cid, err := client.Upload(bytes.NewReader(data), uploadName)
if err != nil { if err != nil {
log.Fatalf("Upload failed: %v", err) log.Fatalf("Upload failed: %v", err)
} }
if err := client.Stop(); err != nil {
log.Printf("Warning: Failed to stop CodexClient: %v", err)
}
if err := client.Destroy(); err != nil {
log.Printf("Warning: Failed to stop CodexClient: %v", err)
}
fmt.Printf("✅ Upload successful!\n") fmt.Printf("✅ Upload successful!\n")
fmt.Printf("CID: %s\n", cid) fmt.Printf("CID: %s\n", cid)
} }

4
communities/codex_archive_downloader.go
View File

@ -316,8 +316,8 @@ func (d *CodexArchiveDownloader) triggerSingleArchiveDownload(hash, cid string,
return fmt.Errorf("failed to trigger archive download with CID %s: %w", cid, err) return fmt.Errorf("failed to trigger archive download with CID %s: %w", cid, err)
} }
if manifest.CID != cid { if manifest.Cid != cid {
return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.CID, cid) return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.Cid, cid)
} }
return nil return nil

23
communities/codex_archive_downloader_integration_test.go
View File

@ -8,10 +8,10 @@ import (
"context" "context"
"crypto/rand" "crypto/rand"
"encoding/hex" "encoding/hex"
"os"
"testing" "testing"
"time" "time"
"github.com/codex-storage/codex-go-bindings/codex"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
@ -31,19 +31,18 @@ type CodexArchiveDownloaderIntegrationSuite struct {
// SetupSuite runs once before all tests in the suite // SetupSuite runs once before all tests in the suite
func (suite *CodexArchiveDownloaderIntegrationSuite) SetupSuite() { func (suite *CodexArchiveDownloaderIntegrationSuite) SetupSuite() {
// Use port 8001 as specified by the user var err error
host := communities.GetEnvOrDefault("CODEX_HOST", "localhost") suite.client, err = communities.NewCodexClient(codex.Config{
port := communities.GetEnvOrDefault("CODEX_API_PORT", "8001") LogFormat: codex.LogFormatNoColors,
suite.client = communities.NewCodexClient(host, port) MetricsEnabled: false,
BlockRetries: 5,
// Optional request timeout override LogLevel: "ERROR",
if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { })
if d, err := time.ParseDuration(ms + "ms"); err == nil { if err != nil {
suite.client.SetRequestTimeout(d) suite.T().Fatalf("Failed to create CodexClient: %v", err)
}
} }
suite.T().Logf("CodexClient configured for %s:%s", host, port) suite.T().Logf("CodexClient configured for")
} }
// TearDownSuite runs once after all tests in the suite // TearDownSuite runs once after all tests in the suite

45
communities/codex_archive_downloader_test.go
View File

@ -7,6 +7,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/codex-storage/codex-go-bindings/codex"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
@ -63,7 +64,7 @@ func (suite *CodexArchiveDownloaderSuite) TestBasicSingleArchive() {
// Set up mock expectations - same as before // Set up mock expectations - same as before
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). Return(codex.Manifest{Cid: "test-cid-1"}, nil).
Times(1) Times(1)
// First HasCid call returns false, second returns true (simulating polling) // First HasCid call returns false, second returns true (simulating polling)
@ -149,7 +150,7 @@ func (suite *CodexArchiveDownloaderSuite) TestMultipleArchives() {
for _, cid := range expectedCids { for _, cid := range expectedCids {
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), cid). TriggerDownloadWithContext(gomock.Any(), cid).
Return(&communities.CodexManifest{CID: cid}, nil). Return(codex.Manifest{Cid: cid}, nil).
Times(1) Times(1)
// Each archive becomes available after one poll // Each archive becomes available after one poll
@ -236,7 +237,7 @@ func (suite *CodexArchiveDownloaderSuite) TestErrorDuringTriggerDownload() {
// Mock TriggerDownloadWithContext to simulate an error // Mock TriggerDownloadWithContext to simulate an error
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
Return(nil, assert.AnError). // Return a generic error to simulate failure Return(codex.Manifest{}, assert.AnError). // Return a generic error to simulate failure
Times(1) Times(1)
// No HasCid calls should be made since TriggerDownload fails // No HasCid calls should be made since TriggerDownload fails
@ -288,13 +289,13 @@ func (suite *CodexArchiveDownloaderSuite) TestActualCancellationDuringTriggerDow
// Use DoAndReturn to create a realistic TriggerDownload that waits for cancellation // Use DoAndReturn to create a realistic TriggerDownload that waits for cancellation
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
// Simulate work by waiting for context cancellation // Simulate work by waiting for context cancellation
select { select {
case <-time.After(5 * time.Second): // This should never happen in our test case <-time.After(5 * time.Second): // This should never happen in our test
return &communities.CodexManifest{CID: cid}, nil return codex.Manifest{Cid: cid}, nil
case <-ctx.Done(): // Wait for actual context cancellation case <-ctx.Done(): // Wait for actual context cancellation
return nil, ctx.Err() // Return the actual cancellation error return codex.Manifest{}, ctx.Err() // Return the actual cancellation error
} }
}). }).
Times(1) Times(1)
@ -352,7 +353,7 @@ func (suite *CodexArchiveDownloaderSuite) TestCancellationDuringPolling() {
// Mock successful TriggerDownload // Mock successful TriggerDownload
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). Return(codex.Manifest{Cid: "test-cid-1"}, nil).
Times(1) Times(1)
// Mock polling - allow multiple calls, but we'll cancel before completion // Mock polling - allow multiple calls, but we'll cancel before completion
@ -420,7 +421,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPollingTimeout() {
// Mock successful TriggerDownload // Mock successful TriggerDownload
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). TriggerDownloadWithContext(gomock.Any(), "test-cid-1").
Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). Return(codex.Manifest{Cid: "test-cid-1"}, nil).
Times(1) Times(1)
// Mock polling to always return false (simulating timeout) // Mock polling to always return false (simulating timeout)
@ -496,7 +497,7 @@ func (suite *CodexArchiveDownloaderSuite) TestWithExistingArchives() {
// Only archive-2 should be downloaded (not in existingArchiveIDs) // Only archive-2 should be downloaded (not in existingArchiveIDs)
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2"). TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(&communities.CodexManifest{CID: "cid-2"}, nil). Return(codex.Manifest{Cid: "cid-2"}, nil).
Times(1) // Only one call expected Times(1) // Only one call expected
// Only archive-2 should be polled // Only archive-2 should be polled
@ -577,7 +578,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_OneSuccessOneError(
// Archive-2 succeeds // Archive-2 succeeds
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2"). TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(&communities.CodexManifest{CID: "cid-2"}, nil) Return(codex.Manifest{Cid: "cid-2"}, nil)
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
HasCid("cid-2"). HasCid("cid-2").
Return(true, nil) Return(true, nil)
@ -585,7 +586,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_OneSuccessOneError(
// Archive-1 fails // Archive-1 fails
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-1"). TriggerDownloadWithContext(gomock.Any(), "cid-1").
Return(nil, fmt.Errorf("trigger failed")) Return(codex.Manifest{}, fmt.Errorf("trigger failed"))
logger := zap.NewNop() logger := zap.NewNop()
downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger) downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger)
@ -633,7 +634,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessErrorCancell
// Archive-3 (newest) succeeds // Archive-3 (newest) succeeds
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-3"). TriggerDownloadWithContext(gomock.Any(), "cid-3").
Return(&communities.CodexManifest{CID: "cid-3"}, nil) Return(codex.Manifest{Cid: "cid-3"}, nil)
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
HasCid("cid-3"). HasCid("cid-3").
Return(true, nil) Return(true, nil)
@ -641,14 +642,14 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessErrorCancell
// Archive-2 fails // Archive-2 fails
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2"). TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(nil, fmt.Errorf("trigger failed")) Return(codex.Manifest{}, fmt.Errorf("trigger failed"))
// Archive-1 will be cancelled (no expectations needed) // Archive-1 will be cancelled (no expectations needed)
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-1"). TriggerDownloadWithContext(gomock.Any(), "cid-1").
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
<-ctx.Done() // Wait for cancellation <-ctx.Done() // Wait for cancellation
return nil, ctx.Err() return codex.Manifest{}, ctx.Err()
}). }).
AnyTimes() AnyTimes()
@ -700,7 +701,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessThenCancella
// Archive-2 (newer) succeeds // Archive-2 (newer) succeeds
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2"). TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(&communities.CodexManifest{CID: "cid-2"}, nil) Return(codex.Manifest{Cid: "cid-2"}, nil)
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
HasCid("cid-2"). HasCid("cid-2").
Return(true, nil) Return(true, nil)
@ -708,9 +709,9 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessThenCancella
// Archive-1 will be cancelled // Archive-1 will be cancelled
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-1"). TriggerDownloadWithContext(gomock.Any(), "cid-1").
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
<-ctx.Done() // Wait for cancellation <-ctx.Done() // Wait for cancellation
return nil, ctx.Err() return codex.Manifest{}, ctx.Err()
}). }).
AnyTimes() AnyTimes()
@ -762,9 +763,9 @@ func (suite *CodexArchiveDownloaderSuite) TestNoSuccess_OnlyCancellation() {
// Both archives will be cancelled // Both archives will be cancelled
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), gomock.Any()). TriggerDownloadWithContext(gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
<-ctx.Done() // Wait for cancellation <-ctx.Done() // Wait for cancellation
return nil, ctx.Err() return codex.Manifest{}, ctx.Err()
}). }).
AnyTimes() AnyTimes()
@ -815,10 +816,10 @@ func (suite *CodexArchiveDownloaderSuite) TestNoSuccess_OnlyErrors() {
// Both archives fail // Both archives fail
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-1"). TriggerDownloadWithContext(gomock.Any(), "cid-1").
Return(nil, fmt.Errorf("trigger failed for cid-1")) Return(codex.Manifest{}, fmt.Errorf("trigger failed for cid-1"))
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
TriggerDownloadWithContext(gomock.Any(), "cid-2"). TriggerDownloadWithContext(gomock.Any(), "cid-2").
Return(nil, fmt.Errorf("trigger failed for cid-2")) Return(codex.Manifest{}, fmt.Errorf("trigger failed for cid-2"))
logger := zap.NewNop() logger := zap.NewNop()
downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger) downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger)

261
communities/codex_client.go
View File

@ -9,63 +9,48 @@ package communities
import ( import (
"bytes" "bytes"
"context" "context"
"encoding/json"
"fmt" "fmt"
"io" "io"
"net/http"
"strings" "github.com/codex-storage/codex-go-bindings/codex"
"time"
) )
// CodexClient handles basic upload/download operations with Codex storage // CodexClient handles basic upload/download operations with Codex storage
type CodexClient struct { type CodexClient struct {
BaseURL string node *codex.CodexNode
Client *http.Client config *codex.Config
} }
// NewCodexClient creates a new Codex client // NewCodexClient creates a new Codex client
func NewCodexClient(host string, port string) *CodexClient { func NewCodexClient(config codex.Config) (*CodexClient, error) {
return &CodexClient{ node, err := codex.New(config)
BaseURL: fmt.Sprintf("http://%s:%s", host, port), if err != nil {
Client: &http.Client{Timeout: 60 * time.Second}, return nil, fmt.Errorf("failed to create Codex node: %w", err)
} }
return &CodexClient{
node: node,
config: &config,
}, nil
}
func (c CodexClient) Start() error {
return c.node.Start()
}
func (c CodexClient) Stop() error {
return c.node.Stop()
}
func (c CodexClient) Destroy() error {
return c.node.Destroy()
} }
// Upload uploads data from a reader to Codex and returns the CID // Upload uploads data from a reader to Codex and returns the CID
func (c *CodexClient) Upload(data io.Reader, filename string) (string, error) { func (c *CodexClient) Upload(data io.Reader, filename string) (string, error) {
url := fmt.Sprintf("%s/api/codex/v1/data", c.BaseURL) return c.node.UploadReader(codex.UploadOptions{
Filepath: filename,
// Create the HTTP request }, data)
req, err := http.NewRequest("POST", url, data)
if err != nil {
return "", fmt.Errorf("failed to create request: %w", err)
}
// Set headers
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Content-Disposition", fmt.Sprintf(`filename="%s"`, filename))
// Send request
resp, err := c.Client.Do(req)
if err != nil {
return "", fmt.Errorf("failed to upload to codex: %w", err)
}
defer resp.Body.Close()
// Check if request was successful
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return "", fmt.Errorf("codex upload failed with status %d: %s", resp.StatusCode, string(body))
}
// Read the CID response
cidBytes, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("failed to read response: %w", err)
}
cid := strings.TrimSpace(string(cidBytes))
return cid, nil
} }
// Download downloads data from Codex by CID and writes it to the provided writer // Download downloads data from Codex by CID and writes it to the provided writer
@ -73,201 +58,43 @@ func (c *CodexClient) Download(cid string, output io.Writer) error {
return c.DownloadWithContext(context.Background(), cid, output) return c.DownloadWithContext(context.Background(), cid, output)
} }
func (c *CodexClient) TriggerDownload(cid string) (*CodexManifest, error) { func (c *CodexClient) TriggerDownload(cid string) (codex.Manifest, error) {
return c.TriggerDownloadWithContext(context.Background(), cid) return c.TriggerDownloadWithContext(context.Background(), cid)
} }
func (c *CodexClient) HasCid(cid string) (bool, error) { func (c *CodexClient) HasCid(cid string) (bool, error) {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/exists", c.BaseURL, cid) err := c.LocalDownload(cid, io.Discard)
return err == nil, nil
resp, err := c.Client.Get(url)
if err != nil {
return false, fmt.Errorf("failed to check cid existence: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return false, fmt.Errorf("cid check failed with status %d: %s", resp.StatusCode, string(body))
}
// Parse JSON response: {"<cid>": <bool>}
var result map[string]bool
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return false, fmt.Errorf("failed to parse response: %w", err)
}
// Validate the CID key matches request
hasCid, exists := result[cid]
if !exists {
return false, fmt.Errorf("response missing CID key %q", cid)
}
return hasCid, nil
} }
func (c *CodexClient) RemoveCid(cid string) error { func (c *CodexClient) RemoveCid(cid string) error {
url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid) return c.node.Delete(cid)
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return fmt.Errorf("failed trying to delete cid: %s, %w", cid, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("cid delete failed with status %d: %s", resp.StatusCode, string(body))
}
return nil
} }
// DownloadWithContext downloads data from Codex by CID with cancellation support // DownloadWithContext downloads data from Codex by CID with cancellation support
func (c *CodexClient) DownloadWithContext(ctx context.Context, cid string, output io.Writer) error { func (c *CodexClient) DownloadWithContext(ctx context.Context, cid string, output io.Writer) error {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/stream", c.BaseURL, cid) return c.node.DownloadStream(cid, codex.DownloadStreamOptions{
Writer: output,
req, err := http.NewRequestWithContext(ctx, "GET", url, nil) })
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return fmt.Errorf("failed to download from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body))
}
// Use context-aware copy for cancellable streaming
return c.copyWithContext(ctx, output, resp.Body)
} }
func (c *CodexClient) LocalDownload(cid string, output io.Writer) error { func (c *CodexClient) LocalDownload(cid string, output io.Writer) error {
return c.LocalDownloadWithContext(context.Background(), cid, output) return c.node.DownloadStream(cid, codex.DownloadStreamOptions{
Writer: output,
Local: true,
})
} }
func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error { func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error {
url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid) return c.LocalDownload(cid, output)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return fmt.Errorf("failed to download from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body))
}
// Use context-aware copy for cancellable streaming
return c.copyWithContext(ctx, output, resp.Body)
} }
func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) { func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error) {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/manifest", c.BaseURL, cid) return c.node.DownloadManifest(cid)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch manifest from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("codex fetch manifest failed with status %d: %s", resp.StatusCode, string(body))
}
// Parse JSON response containing manifest
var manifest CodexManifest
if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil {
return nil, fmt.Errorf("failed to parse manifest: %w", err)
}
return &manifest, nil
} }
func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) { func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error) {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/network", c.BaseURL, cid) return c.node.Fetch(cid)
req, err := http.NewRequestWithContext(ctx, "POST", url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to trigger download from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("codex async download failed with status %d: %s", resp.StatusCode, string(body))
}
// Parse JSON response containing manifest
var manifest CodexManifest
if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil {
return nil, fmt.Errorf("failed to parse download manifest: %w", err)
}
return &manifest, nil
}
// copyWithContext performs io.Copy but respects context cancellation
func (c *CodexClient) copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error {
// Create a buffer for chunked copying
buf := make([]byte, 64*1024) // 64KB buffer
for {
select {
case <-ctx.Done():
return ctx.Err() // Return cancellation error
default:
}
// Read a chunk
n, err := src.Read(buf)
if n > 0 {
// Write the chunk
if _, writeErr := dst.Write(buf[:n]); writeErr != nil {
return fmt.Errorf("failed to write data: %w", writeErr)
}
}
if err == io.EOF {
return nil // Successful completion
}
if err != nil {
return fmt.Errorf("failed to read data: %w", err)
}
}
}
// SetRequestTimeout sets the HTTP client timeout for requests
func (c *CodexClient) SetRequestTimeout(timeout time.Duration) {
c.Client.Timeout = timeout
} }
// UploadArchive is a convenience method for uploading archive data // UploadArchive is a convenience method for uploading archive data

63
communities/codex_client_integration_test.go
View File

@ -8,10 +8,10 @@ import (
"context" "context"
"crypto/rand" "crypto/rand"
"encoding/hex" "encoding/hex"
"os"
"testing" "testing"
"time" "time"
"github.com/codex-storage/codex-go-bindings/codex"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
@ -20,29 +20,22 @@ import (
) )
// CodexClientIntegrationTestSuite demonstrates testify's suite functionality for CodexClient integration tests // CodexClientIntegrationTestSuite demonstrates testify's suite functionality for CodexClient integration tests
// These tests exercise real network calls against a running Codex node.
// Required env vars (with defaults):
// - CODEX_HOST (default: localhost)
// - CODEX_API_PORT (default: 8080)
// - CODEX_TIMEOUT_MS (optional; default: 60000)
type CodexClientIntegrationTestSuite struct { type CodexClientIntegrationTestSuite struct {
suite.Suite suite.Suite
client *communities.CodexClient client *communities.CodexClient
host string
port string
} }
// SetupSuite runs once before all tests in the suite // SetupSuite runs once before all tests in the suite
func (suite *CodexClientIntegrationTestSuite) SetupSuite() { func (suite *CodexClientIntegrationTestSuite) SetupSuite() {
suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost") var err error
suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8080") suite.client, err = communities.NewCodexClient(codex.Config{
suite.client = communities.NewCodexClient(suite.host, suite.port) DataDir: suite.T().TempDir(),
LogFormat: codex.LogFormatNoColors,
// Optional request timeout override MetricsEnabled: false,
if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { BlockRetries: 5,
if d, err := time.ParseDuration(ms + "ms"); err == nil { })
suite.client.SetRequestTimeout(d) if err != nil {
} suite.T().Fatalf("Failed to create Codex client: %v", err)
} }
} }
@ -114,15 +107,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_CheckNonExistingCI
} }
func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload() { func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload() {
// Use port 8001 for this test as specified client := communities.NewCodexClientTest(suite.T())
client := communities.NewCodexClient(suite.host, "8001")
// Optional request timeout override
if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" {
if d, err := time.ParseDuration(ms + "ms"); err == nil {
client.SetRequestTimeout(d)
}
}
// Generate random payload to ensure proper round-trip verification // Generate random payload to ensure proper round-trip verification
payload := make([]byte, 1024) payload := make([]byte, 1024)
@ -145,7 +130,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload()
// Trigger async download // Trigger async download
manifest, err := client.TriggerDownload(cid) manifest, err := client.TriggerDownload(cid)
require.NoError(suite.T(), err, "TriggerDownload failed") require.NoError(suite.T(), err, "TriggerDownload failed")
suite.T().Logf("Async download triggered, manifest CID: %s", manifest.CID) suite.T().Logf("Async download triggered, manifest CID: %s", manifest.Cid)
// Poll HasCid for up to 10 seconds using goroutine and channel // Poll HasCid for up to 10 seconds using goroutine and channel
downloadComplete := make(chan bool, 1) downloadComplete := make(chan bool, 1)
@ -221,27 +206,27 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_FetchManifest() {
manifest, err := suite.client.FetchManifestWithContext(ctx, cid) manifest, err := suite.client.FetchManifestWithContext(ctx, cid)
require.NoError(suite.T(), err, "FetchManifestWithContext failed") require.NoError(suite.T(), err, "FetchManifestWithContext failed")
suite.T().Logf("FetchManifest successful, manifest CID: %s", manifest.CID) suite.T().Logf("FetchManifest successful, manifest CID: %s", manifest.Cid)
// Verify manifest properties // Verify manifest properties
assert.Equal(suite.T(), cid, manifest.CID, "Manifest CID mismatch") assert.Equal(suite.T(), cid, manifest.Cid, "Manifest CID mismatch")
// Verify manifest has expected fields // Verify manifest has expected fields
assert.NotEmpty(suite.T(), manifest.Manifest.TreeCid, "Expected TreeCid to be non-empty") assert.NotEmpty(suite.T(), manifest.TreeCid, "Expected TreeCid to be non-empty")
suite.T().Logf("Manifest TreeCid: %s", manifest.Manifest.TreeCid) suite.T().Logf("Manifest TreeCid: %s", manifest.TreeCid)
assert.Greater(suite.T(), manifest.Manifest.DatasetSize, int64(0), "Expected DatasetSize > 0") assert.Greater(suite.T(), manifest.DatasetSize, 0, "Expected DatasetSize > 0")
suite.T().Logf("Manifest DatasetSize: %d", manifest.Manifest.DatasetSize) suite.T().Logf("Manifest DatasetSize: %d", manifest.DatasetSize)
assert.Greater(suite.T(), manifest.Manifest.BlockSize, 0, "Expected BlockSize > 0") assert.Greater(suite.T(), manifest.BlockSize, 0, "Expected BlockSize > 0")
suite.T().Logf("Manifest BlockSize: %d", manifest.Manifest.BlockSize) suite.T().Logf("Manifest BlockSize: %d", manifest.BlockSize)
assert.Equal(suite.T(), "fetch-manifest-test.bin", manifest.Manifest.Filename, "Filename mismatch") assert.Equal(suite.T(), "fetch-manifest-test.bin", manifest.Filename, "Filename mismatch")
suite.T().Logf("Manifest Filename: %s", manifest.Manifest.Filename) suite.T().Logf("Manifest Filename: %s", manifest.Filename)
// Log manifest details for verification // Log manifest details for verification
suite.T().Logf("Manifest Protected: %v", manifest.Manifest.Protected) suite.T().Logf("Manifest Protected: %v", manifest.Protected)
suite.T().Logf("Manifest Mimetype: %s", manifest.Manifest.Mimetype) suite.T().Logf("Manifest Mimetype: %s", manifest.Mimetype)
// Test fetching manifest for non-existent CID (should fail gracefully) // Test fetching manifest for non-existent CID (should fail gracefully)
nonExistentCID := "zDvZRwzmNonExistentCID123456789" nonExistentCID := "zDvZRwzmNonExistentCID123456789"

12
communities/codex_client_interface.go
View File

@ -3,7 +3,8 @@ package communities
import ( import (
"context" "context"
"io" "io"
"time"
"github.com/codex-storage/codex-go-bindings/codex"
) )
// Mock generation instruction above will create a mock in package `mock_communities` // Mock generation instruction above will create a mock in package `mock_communities`
@ -25,16 +26,13 @@ type CodexClientInterface interface {
LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error
// Async download methods // Async download methods
TriggerDownload(cid string) (*CodexManifest, error) TriggerDownload(cid string) (codex.Manifest, error)
TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error)
// Manifest methods // Manifest methods
FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error)
// CID management methods // CID management methods
HasCid(cid string) (bool, error) HasCid(cid string) (bool, error)
RemoveCid(cid string) error RemoveCid(cid string) error
// Configuration methods
SetRequestTimeout(timeout time.Duration)
} }

556
communities/codex_client_test.go
View File

@ -4,10 +4,7 @@ import (
"bytes" "bytes"
"context" "context"
"errors" "errors"
"fmt"
"io" "io"
"net/http"
"net/http/httptest"
"testing" "testing"
"time" "time"
@ -18,24 +15,33 @@ import (
"go-codex-client/communities" "go-codex-client/communities"
) )
func upload(client communities.CodexClient, t *testing.T, buf *bytes.Buffer) string {
filename := "hello.txt"
cid, err := client.Upload(buf, filename)
if err != nil {
t.Fatalf("Failed to upload file: %v", err)
}
if cid == "" {
t.Fatalf("Expected non-empty CID after upload")
}
return cid
}
// CodexClientTestSuite demonstrates testify's suite functionality for CodexClient tests // CodexClientTestSuite demonstrates testify's suite functionality for CodexClient tests
type CodexClientTestSuite struct { type CodexClientTestSuite struct {
suite.Suite suite.Suite
client *communities.CodexClient client *communities.CodexClient
server *httptest.Server
} }
// SetupTest runs before each test method // SetupTest runs before each test method
func (suite *CodexClientTestSuite) SetupTest() { func (suite *CodexClientTestSuite) SetupTest() {
suite.client = communities.NewCodexClient("localhost", "8080") suite.client = communities.NewCodexClientTest(suite.T())
} }
// TearDownTest runs after each test method // TearDownTest runs after each test method
func (suite *CodexClientTestSuite) TearDownTest() { func (suite *CodexClientTestSuite) TearDownTest() {
if suite.server != nil {
suite.server.Close()
suite.server = nil
}
} }
// TestCodexClientTestSuite runs the test suite // TestCodexClientTestSuite runs the test suite
@ -44,110 +50,33 @@ func TestCodexClientTestSuite(t *testing.T) {
} }
func (suite *CodexClientTestSuite) TestUpload_Success() { func (suite *CodexClientTestSuite) TestUpload_Success() {
// Arrange a fake Codex server that validates headers and returns a CID
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if r.URL.Path != "/api/codex/v1/data" {
w.WriteHeader(http.StatusNotFound)
return
}
if ct := r.Header.Get("Content-Type"); ct != "application/octet-stream" {
w.WriteHeader(http.StatusBadRequest)
return
}
if cd := r.Header.Get("Content-Disposition"); cd != "filename=\"hello.txt\"" {
w.WriteHeader(http.StatusBadRequest)
return
}
_, _ = io.ReadAll(r.Body) // consume body
_ = r.Body.Close()
w.WriteHeader(http.StatusOK)
// Codex returns CIDv1 base58btc
// prefix: zDv
// - z = multibase prefix for base58btc
// - Dv = CIDv1 prefix for raw codex
// we add a newline to simulate real response
_, _ = w.Write([]byte("zDvZRwzmTestCID123\n"))
}))
suite.client.BaseURL = suite.server.URL
// Act // Act
cid, err := suite.client.Upload(bytes.NewReader([]byte("payload")), "hello.txt") cid, err := suite.client.Upload(bytes.NewReader([]byte("payload")), "hello.txt")
// Assert // Assert
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
// Codex uses CIDv1 with base58btc encoding (prefix: zDv) // Codex uses CIDv1 with base58btc encoding (prefix: zDv)
assert.Equal(suite.T(), "zDvZRwzmTestCID123", cid) assert.Equal(suite.T(), "zDvZRwzmBEaJ338xaCHbKbGAJ4X41YyccS6eyorrYBbmPnWuLxCh", cid)
}
func (suite *CodexClientTestSuite) TestDownload_Success() {
const wantCID = "zDvZRwzm"
const payload = "hello from codex"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if r.URL.Path != "/api/codex/v1/data/"+wantCID+"/network/stream" {
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(payload))
}))
suite.client.BaseURL = suite.server.URL
var buf bytes.Buffer
err := suite.client.Download(wantCID, &buf)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), payload, buf.String())
} }
func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() { func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() {
const cid = "zDvZRwzm" // skip test
suite.T().Skip("Wait for cancellation support PR to be merged in codex-go-bindings")
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/codex/v1/data/"+cid+"/network/stream" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/octet-stream")
flusher, _ := w.(http.Flusher)
w.WriteHeader(http.StatusOK)
// Stream data slowly so the request can be canceled
for i := 0; i < 1000; i++ {
select {
case <-r.Context().Done():
return
default:
}
if _, err := w.Write([]byte("x")); err != nil {
// Client likely went away; stop writing
return
}
if flusher != nil {
flusher.Flush()
}
time.Sleep(10 * time.Millisecond)
}
}))
suite.client.BaseURL = suite.server.URL
len := 1024 * 1024 * 50
buf := bytes.NewBuffer(make([]byte, len))
cid := upload(*suite.client, suite.T(), buf)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
err := suite.client.DownloadWithContext(ctx, cid, io.Discard) channelError := make(chan error, 1)
go func() {
err := suite.client.DownloadWithContext(ctx, cid, io.Discard)
channelError <- err
}()
cancel()
err := <-channelError
require.Error(suite.T(), err) require.Error(suite.T(), err)
// Accept either canceled or deadline exceeded depending on timing // Accept either canceled or deadline exceeded depending on timing
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
@ -160,31 +89,20 @@ func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() {
} }
func (suite *CodexClientTestSuite) TestHasCid_Success() { func (suite *CodexClientTestSuite) TestHasCid_Success() {
const payload = "hello from codex"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
tests := []struct { tests := []struct {
name string name string
cid string cid string
hasIt bool
wantBool bool wantBool bool
}{ }{
{"has CID returns true", "zDvZRwzmTestCID", true, true}, {"has CID returns true", cid, true},
{"has CID returns false", "zDvZRwzmTestCID", false, false}, {"has CID returns false", "zDvZRwzmBEaJ338xaCHbKbGAJ4X41YyccS6eyorrYBbmPnWuLxCe", false},
} }
for _, tt := range tests { for _, tt := range tests {
suite.Run(tt.name, func() { suite.Run(tt.name, func() {
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/codex/v1/data/"+tt.cid+"/exists" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
// Return JSON: {"<cid>": <bool>}
fmt.Fprintf(w, `{"%s": %t}`, tt.cid, tt.hasIt)
}))
suite.client.BaseURL = suite.server.URL
got, err := suite.client.HasCid(tt.cid) got, err := suite.client.HasCid(tt.cid)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
assert.Equal(suite.T(), tt.wantBool, got, "HasCid(%q) = %v, want %v", tt.cid, got, tt.wantBool) assert.Equal(suite.T(), tt.wantBool, got, "HasCid(%q) = %v, want %v", tt.cid, got, tt.wantBool)
@ -192,181 +110,42 @@ func (suite *CodexClientTestSuite) TestHasCid_Success() {
} }
} }
func (suite *CodexClientTestSuite) TestHasCid_RequestError() { func (suite *CodexClientTestSuite) TestDownload_Success() {
// Create a server and immediately close it to trigger connection error const payload = "hello from codex"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
suite.server.Close() // Close immediately so connection fails
suite.client.BaseURL = suite.server.URL // Use the closed server's URL var buf bytes.Buffer
err := suite.client.Download(cid, &buf)
got, err := suite.client.HasCid("zDvZRwzmTestCID") require.NoError(suite.T(), err)
require.Error(suite.T(), err) assert.Equal(suite.T(), payload, buf.String())
assert.False(suite.T(), got, "expected false on error")
}
func (suite *CodexClientTestSuite) TestHasCid_CidMismatch() {
const requestCid = "zDvZRwzmRequestCID"
const responseCid = "zDvZRwzmDifferentCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
// Return a different CID in the response
fmt.Fprintf(w, `{"%s": true}`, responseCid)
}))
suite.client.BaseURL = suite.server.URL
got, err := suite.client.HasCid(requestCid)
require.Error(suite.T(), err, "expected error for CID mismatch")
assert.False(suite.T(), got, "expected false on CID mismatch")
// Check error message mentions the missing/mismatched CID
assert.Contains(suite.T(), err.Error(), requestCid, "error should mention request CID")
} }
func (suite *CodexClientTestSuite) TestRemoveCid_Success() { func (suite *CodexClientTestSuite) TestRemoveCid_Success() {
const testCid = "zDvZRwzmTestCID" const payload = "hello from codex"
cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { err := suite.client.RemoveCid(cid)
if r.Method != http.MethodDelete {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if r.URL.Path != "/api/codex/v1/data/"+testCid {
w.WriteHeader(http.StatusNotFound)
return
}
// DELETE should return 204 No Content
w.WriteHeader(http.StatusNoContent)
}))
suite.client.BaseURL = suite.server.URL
err := suite.client.RemoveCid(testCid)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
} }
func (suite *CodexClientTestSuite) TestRemoveCid_Error() {
const testCid = "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Return error status
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("server error"))
}))
suite.client.BaseURL = suite.server.URL
err := suite.client.RemoveCid(testCid)
require.Error(suite.T(), err)
assert.Contains(suite.T(), err.Error(), "500", "error should mention status 500")
}
func (suite *CodexClientTestSuite) TestTriggerDownload() { func (suite *CodexClientTestSuite) TestTriggerDownload() {
const testCid = "zDvZRwzmTestCID" const payload = "hello from codex"
const expectedManifest = `{ cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
"cid": "zDvZRwzmTestCID",
"manifest": {
"treeCid": "zDvZRwzmTreeCID",
"datasetSize": 1024,
"blockSize": 65536,
"protected": false,
"filename": "test-file.bin",
"mimetype": "application/octet-stream"
}
}`
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusMethodNotAllowed)
return
}
if r.URL.Path != "/api/codex/v1/data/"+testCid+"/network" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(expectedManifest))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background() ctx := context.Background()
manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid) manifest, err := suite.client.TriggerDownloadWithContext(ctx, cid)
require.NoError(suite.T(), err) require.NoError(suite.T(), err)
assert.Equal(suite.T(), testCid, manifest.CID) assert.Equal(suite.T(), cid, manifest.Cid)
assert.Equal(suite.T(), "zDvZRwzmTreeCID", manifest.Manifest.TreeCid) assert.Equal(suite.T(), "zDzSvJTf7mGkC3yuiVGco7Qc6s4LA8edye9inT4w2QqHnfbuRvMr", manifest.TreeCid)
assert.Equal(suite.T(), int64(1024), manifest.Manifest.DatasetSize) assert.Equal(suite.T(), len(payload), manifest.DatasetSize)
assert.Equal(suite.T(), "test-file.bin", manifest.Manifest.Filename) assert.Equal(suite.T(), "hello.txt", manifest.Filename)
}
func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_RequestError() {
// Create a server and immediately close it to trigger connection error
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
suite.server.Close()
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.TriggerDownloadWithContext(ctx, "zDvZRwzmRigWseNB7WqmudkKAPgZmrDCE9u5cY4KvCqhRo9Ki")
require.Error(suite.T(), err)
assert.Nil(suite.T(), manifest, "expected nil manifest on error")
}
func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_JSONParseError() {
const testCid = "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
// Return invalid JSON
w.Write([]byte(`{"invalid": json}`))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid)
require.Error(suite.T(), err, "expected JSON parse error")
assert.Nil(suite.T(), manifest, "expected nil manifest on parse error")
assert.Contains(suite.T(), err.Error(), "failed to parse download manifest", "error should mention parse failure")
}
func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_HTTPError() {
const testCid = "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("CID not found"))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid)
require.Error(suite.T(), err, "expected error for 404 status")
assert.Nil(suite.T(), manifest, "expected nil manifest on HTTP error")
assert.Contains(suite.T(), err.Error(), "404", "error should mention status 404")
} }
func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation() { func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation() {
suite.T().Skip("Not sure if we are going to have cancellation in trigger download")
const testCid = "zDvZRwzmTestCID" const testCid = "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Simulate slow response to allow cancellation
select {
case <-r.Context().Done():
return
case <-time.After(200 * time.Millisecond):
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"cid": "test"}`))
}
}))
suite.client.BaseURL = suite.server.URL
// Cancel after 50ms (before server responds) // Cancel after 50ms (before server responds)
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel() defer cancel()
@ -385,99 +164,43 @@ func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation()
} }
func (suite *CodexClientTestSuite) TestLocalDownload() { func (suite *CodexClientTestSuite) TestLocalDownload() {
testData := []byte("test data for local download") const payload = "test data for local download"
testCid := "zDvZRwzmTestCID" cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Verify request method and path
assert.Equal(suite.T(), "GET", r.Method, "Expected GET request")
expectedPath := "/api/codex/v1/data/" + testCid
assert.Equal(suite.T(), expectedPath, r.URL.Path, "Expected correct path")
w.WriteHeader(http.StatusOK)
w.Write(testData)
}))
suite.client.BaseURL = suite.server.URL
var buf bytes.Buffer var buf bytes.Buffer
err := suite.client.LocalDownload(testCid, &buf) err := suite.client.LocalDownload(cid, &buf)
require.NoError(suite.T(), err, "LocalDownload failed") require.NoError(suite.T(), err, "LocalDownload failed")
assert.Equal(suite.T(), testData, buf.Bytes(), "Downloaded data mismatch") assert.Equal(suite.T(), payload, buf.String(), "Downloaded data mismatch")
} }
func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Success() { func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Success() {
testData := []byte("test data for local download with context") const payload = "test data for local download with context"
testCid := "zDvZRwzmTestCID" cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Verify request method and path
assert.Equal(suite.T(), "GET", r.Method, "Expected GET request")
expectedPath := "/api/codex/v1/data/" + testCid
assert.Equal(suite.T(), expectedPath, r.URL.Path, "Expected correct path")
w.WriteHeader(http.StatusOK)
w.Write(testData)
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background() ctx := context.Background()
var buf bytes.Buffer var buf bytes.Buffer
err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf) err := suite.client.LocalDownloadWithContext(ctx, cid, &buf)
require.NoError(suite.T(), err, "LocalDownloadWithContext failed") require.NoError(suite.T(), err, "LocalDownloadWithContext failed")
assert.Equal(suite.T(), testData, buf.Bytes(), "Downloaded data mismatch") assert.Equal(suite.T(), payload, buf.String(), "Downloaded data mismatch")
}
func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_RequestError() {
// Create a server and immediately close it to trigger connection error
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
suite.server.Close()
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
var buf bytes.Buffer
err := suite.client.LocalDownloadWithContext(ctx, "zDvZRwzmTestCID", &buf)
require.Error(suite.T(), err, "Expected error due to closed server")
assert.Contains(suite.T(), err.Error(), "failed to download from codex")
}
func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_HTTPError() {
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("CID not found in local storage"))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
var buf bytes.Buffer
err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf)
require.Error(suite.T(), err, "Expected error for HTTP 404")
assert.Contains(suite.T(), err.Error(), "404", "Expected '404' in error message")
} }
func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() { func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() {
testCid := "zDvZRwzmTestCID" suite.T().Skip("Wait for cancellation support PR to be merged in codex-go-bindings")
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { len := 1024 * 1024 * 50
// Simulate a slow response buf := bytes.NewBuffer(make([]byte, len))
time.Sleep(100 * time.Millisecond) cid := upload(*suite.client, suite.T(), buf)
w.WriteHeader(http.StatusOK)
w.Write([]byte("slow response"))
}))
suite.client.BaseURL = suite.server.URL
// Create a context with a very short timeout
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel()
var buf bytes.Buffer channelError := make(chan error, 1)
err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf) go func() {
err := suite.client.LocalDownloadWithContext(ctx, cid, io.Discard)
channelError <- err
}()
cancel()
err := <-channelError
require.Error(suite.T(), err, "Expected context cancellation error") require.Error(suite.T(), err, "Expected context cancellation error")
// Accept either canceled or deadline exceeded depending on timing // Accept either canceled or deadline exceeded depending on timing
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
@ -490,107 +213,28 @@ func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() {
} }
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Success() { func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Success() {
testCid := "zDvZRwzmTestCID" const payload = "hello from codex"
expectedManifest := `{ cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload)))
"cid": "zDvZRwzmTestCID",
"manifest": {
"treeCid": "zDvZRwzmTreeCID123",
"datasetSize": 1024,
"blockSize": 256,
"protected": true,
"filename": "test-file.bin",
"mimetype": "application/octet-stream"
}
}`
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(suite.T(), http.MethodGet, r.Method)
expectedPath := fmt.Sprintf("/api/codex/v1/data/%s/network/manifest", testCid)
assert.Equal(suite.T(), expectedPath, r.URL.Path)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(expectedManifest))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background() ctx := context.Background()
manifest, err := suite.client.FetchManifestWithContext(ctx, testCid) manifest, err := suite.client.FetchManifestWithContext(ctx, cid)
require.NoError(suite.T(), err, "Expected no error") require.NoError(suite.T(), err, "Expected no error")
require.NotNil(suite.T(), manifest, "Expected manifest, got nil") require.NotNil(suite.T(), manifest, "Expected manifest, got nil")
assert.Equal(suite.T(), testCid, manifest.CID) assert.Equal(suite.T(), cid, manifest.Cid)
assert.Equal(suite.T(), "zDvZRwzmTreeCID123", manifest.Manifest.TreeCid) assert.Equal(suite.T(), "zDzSvJTf7mGkC3yuiVGco7Qc6s4LA8edye9inT4w2QqHnfbuRvMr", manifest.TreeCid)
assert.Equal(suite.T(), int64(1024), manifest.Manifest.DatasetSize) assert.Equal(suite.T(), len(payload), manifest.DatasetSize)
assert.Equal(suite.T(), 256, manifest.Manifest.BlockSize) assert.Equal(suite.T(), 65536, manifest.BlockSize)
assert.True(suite.T(), manifest.Manifest.Protected, "Expected Protected to be true") assert.True(suite.T(), !manifest.Protected, "Expected Protected to be false")
assert.Equal(suite.T(), "test-file.bin", manifest.Manifest.Filename) assert.Equal(suite.T(), "hello.txt", manifest.Filename)
assert.Equal(suite.T(), "application/octet-stream", manifest.Manifest.Mimetype) assert.Equal(suite.T(), "text/plain", manifest.Mimetype)
}
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_RequestError() {
// Create a server and immediately close it to trigger connection error
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
suite.server.Close()
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.FetchManifestWithContext(ctx, "test-cid")
require.Error(suite.T(), err, "Expected error for closed server")
assert.Nil(suite.T(), manifest, "Expected nil manifest on error")
assert.Contains(suite.T(), err.Error(), "failed to fetch manifest from codex")
}
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_HTTPError() {
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("Manifest not found"))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.FetchManifestWithContext(ctx, testCid)
require.Error(suite.T(), err, "Expected error for HTTP 404")
assert.Nil(suite.T(), manifest, "Expected nil manifest on error")
assert.Contains(suite.T(), err.Error(), "404", "Expected '404' in error message")
}
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_JSONParseError() {
testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte("invalid json {"))
}))
suite.client.BaseURL = suite.server.URL
ctx := context.Background()
manifest, err := suite.client.FetchManifestWithContext(ctx, testCid)
require.Error(suite.T(), err, "Expected error for invalid JSON")
assert.Nil(suite.T(), manifest, "Expected nil manifest on JSON parse error")
assert.Contains(suite.T(), err.Error(), "failed to parse manifest", "Expected 'failed to parse manifest' in error message")
} }
func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() { func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() {
suite.T().Skip("Not sure if we are going to have cancellation in fetch manifest")
testCid := "zDvZRwzmTestCID" testCid := "zDvZRwzmTestCID"
suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Simulate a slow response
time.Sleep(100 * time.Millisecond)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"cid": "test"}`))
}))
suite.client.BaseURL = suite.server.URL
// Create a context with a very short timeout // Create a context with a very short timeout
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel() defer cancel()
@ -607,4 +251,30 @@ func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() {
suite.T().Fatalf("expected context cancellation, got: %v", err) suite.T().Fatalf("expected context cancellation, got: %v", err)
} }
} }
buf := bytes.NewBuffer([]byte("Hello World!"))
if buf.Len() != manifest.DatasetSize {
suite.T().Errorf("expected size %d, got %d", buf.Len(), manifest.DatasetSize)
}
defaultBlockSize := 1024 * 64
if manifest.BlockSize != defaultBlockSize {
suite.T().Errorf("expected block size %d, got %d", defaultBlockSize, manifest.BlockSize)
}
if manifest.Filename != "test.txt" {
suite.T().Errorf("expected filename %q, got %q", "test.txt", manifest.Filename)
}
if manifest.Protected {
suite.T().Errorf("expected protected to be false, got true")
}
if manifest.Mimetype != "text/plain" {
suite.T().Errorf("expected mimetype %q, got %q", "text/plain", manifest.Mimetype)
}
if manifest.TreeCid == "" {
suite.T().Errorf("expected non-empty TreeCid")
}
} }

8
communities/codex_index_downloader.go
View File

@ -79,19 +79,19 @@ func (d *CodexIndexDownloader) GotManifest() <-chan struct{} {
} }
// Verify that the CID matches our configured indexCid // Verify that the CID matches our configured indexCid
if manifest.CID != d.indexCid { if manifest.Cid != d.indexCid {
d.mu.Lock() d.mu.Lock()
d.downloadError = fmt.Errorf("manifest CID mismatch: expected %s, got %s", d.indexCid, manifest.CID) d.downloadError = fmt.Errorf("manifest CID mismatch: expected %s, got %s", d.indexCid, manifest.Cid)
d.mu.Unlock() d.mu.Unlock()
d.logger.Debug("manifest CID mismatch", d.logger.Debug("manifest CID mismatch",
zap.String("expected", d.indexCid), zap.String("expected", d.indexCid),
zap.String("got", manifest.CID)) zap.String("got", manifest.Cid))
return return
} }
// Store the dataset size for later use - this indicates success // Store the dataset size for later use - this indicates success
d.mu.Lock() d.mu.Lock()
d.datasetSize = manifest.Manifest.DatasetSize d.datasetSize = int64(manifest.DatasetSize)
d.mu.Unlock() d.mu.Unlock()
// Success! Close the channel to signal completion // Success! Close the channel to signal completion

13
communities/codex_index_downloader_integration_test.go
View File

@ -30,23 +30,12 @@ type CodexIndexDownloaderIntegrationTestSuite struct {
suite.Suite suite.Suite
client *communities.CodexClient client *communities.CodexClient
testDir string testDir string
host string
port string
logger *zap.Logger logger *zap.Logger
} }
// SetupSuite runs once before all tests in the suite // SetupSuite runs once before all tests in the suite
func (suite *CodexIndexDownloaderIntegrationTestSuite) SetupSuite() { func (suite *CodexIndexDownloaderIntegrationTestSuite) SetupSuite() {
suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost") suite.client = communities.NewCodexClientTest(suite.T())
suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8001")
suite.client = communities.NewCodexClient(suite.host, suite.port)
// Optional request timeout override
if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" {
if d, err := time.ParseDuration(ms + "ms"); err == nil {
suite.client.SetRequestTimeout(d)
}
}
// Create logger // Create logger
suite.logger, _ = zap.NewDevelopment() suite.logger, _ = zap.NewDevelopment()

43
communities/codex_index_downloader_test.go
View File

@ -9,6 +9,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/codex-storage/codex-go-bindings/codex"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
@ -79,12 +80,12 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_SuccessClosesChannel
filePath := filepath.Join(suite.testDir, "index.bin") filePath := filepath.Join(suite.testDir, "index.bin")
// Setup mock to return a successful manifest // Setup mock to return a successful manifest
expectedManifest := &communities.CodexManifest{ expectedManifest := codex.Manifest{
CID: testCid, Cid: testCid,
} }
expectedManifest.Manifest.DatasetSize = 1024 expectedManifest.DatasetSize = 1024
expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID" expectedManifest.TreeCid = "zDvZRwzmTreeCID"
expectedManifest.Manifest.BlockSize = 65536 expectedManifest.BlockSize = 65536
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid). FetchManifestWithContext(gomock.Any(), testCid).
@ -119,7 +120,7 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_ErrorDoesNotCloseCha
// Setup mock to return an error // Setup mock to return an error
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid). FetchManifestWithContext(gomock.Any(), testCid).
Return(nil, errors.New("fetch error")) Return(codex.Manifest{}, errors.New("fetch error"))
// Create downloader // Create downloader
downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger) downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
@ -154,10 +155,10 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_CidMismatchDoesNotCl
filePath := filepath.Join(suite.testDir, "index.bin") filePath := filepath.Join(suite.testDir, "index.bin")
// Setup mock to return a manifest with different CID // Setup mock to return a manifest with different CID
mismatchedManifest := &communities.CodexManifest{ mismatchedManifest := codex.Manifest{
CID: differentCid, // Different CID! Cid: differentCid, // Different CID!
} }
mismatchedManifest.Manifest.DatasetSize = 1024 mismatchedManifest.DatasetSize = 1024
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid). FetchManifestWithContext(gomock.Any(), testCid).
@ -198,12 +199,12 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_Cancellation() {
fetchCalled := make(chan struct{}) fetchCalled := make(chan struct{})
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid). FetchManifestWithContext(gomock.Any(), testCid).
DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) {
close(fetchCalled) // Signal that fetch was called close(fetchCalled) // Signal that fetch was called
// Wait for context cancellation // Wait for context cancellation
<-ctx.Done() <-ctx.Done()
return nil, ctx.Err() return codex.Manifest{}, ctx.Err()
}) })
// Create downloader // Create downloader
@ -250,11 +251,11 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize()
expectedSize := int64(2048) expectedSize := int64(2048)
// Setup mock to return a manifest with specific dataset size // Setup mock to return a manifest with specific dataset size
expectedManifest := &communities.CodexManifest{ expectedManifest := codex.Manifest{
CID: testCid, Cid: testCid,
} }
expectedManifest.Manifest.DatasetSize = expectedSize expectedManifest.DatasetSize = int(expectedSize)
expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID" expectedManifest.TreeCid = "zDvZRwzmTreeCID"
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid). FetchManifestWithContext(gomock.Any(), testCid).
@ -278,7 +279,7 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize()
} }
// Verify dataset size was recorded correctly // Verify dataset size was recorded correctly
assert.Equal(suite.T(), expectedSize, downloader.GetDatasetSize(), "Dataset size should match manifest") assert.Equal(suite.T(), int64(expectedSize), downloader.GetDatasetSize(), "Dataset size should match manifest")
suite.T().Logf("✅ Dataset size correctly recorded: %d", downloader.GetDatasetSize()) suite.T().Logf("✅ Dataset size correctly recorded: %d", downloader.GetDatasetSize())
// Verify no error was recorded // Verify no error was recorded
@ -500,13 +501,13 @@ func (suite *CodexIndexDownloaderTestSuite) TestDownloadIndexFile_ErrorHandling(
func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() { func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() {
testCid := "zDvZRwzmTestCID123" testCid := "zDvZRwzmTestCID123"
filePath := filepath.Join(suite.testDir, "index.bin") filePath := filepath.Join(suite.testDir, "index.bin")
expectedSize := int64(4096) expectedSize := 4096
// Setup mock to return a manifest // Setup mock to return a manifest
expectedManifest := &communities.CodexManifest{ expectedManifest := codex.Manifest{
CID: testCid, Cid: testCid,
} }
expectedManifest.Manifest.DatasetSize = expectedSize expectedManifest.DatasetSize = expectedSize
suite.mockClient.EXPECT(). suite.mockClient.EXPECT().
FetchManifestWithContext(gomock.Any(), testCid). FetchManifestWithContext(gomock.Any(), testCid).
@ -523,6 +524,6 @@ func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() {
<-manifestChan <-manifestChan
// Now Length should return the dataset size // Now Length should return the dataset size
assert.Equal(suite.T(), expectedSize, downloader.Length(), "Length should return dataset size") assert.Equal(suite.T(), int64(expectedSize), downloader.Length(), "Length should return dataset size")
suite.T().Logf("✅ Length() correctly returns dataset size: %d", downloader.Length()) suite.T().Logf("✅ Length() correctly returns dataset size: %d", downloader.Length())
} }

14
communities/codex_manifest.go
View File

@ -1,14 +0,0 @@
package communities
// CodexManifest represents the manifest structure returned by Codex API
type CodexManifest struct {
CID string `json:"cid"`
Manifest struct {
TreeCid string `json:"treeCid"`
DatasetSize int64 `json:"datasetSize"`
BlockSize int `json:"blockSize"`
Protected bool `json:"protected"`
Filename string `json:"filename"`
Mimetype string `json:"mimetype"`
} `json:"manifest"`
}

14
communities/mock/codex_client_interface.go
View File

@ -11,11 +11,11 @@ package mock_communities
import ( import (
context "context" context "context"
communities "go-codex-client/communities"
io "io" io "io"
reflect "reflect" reflect "reflect"
time "time" time "time"
"github.com/codex-storage/codex-go-bindings/codex"
gomock "go.uber.org/mock/gomock" gomock "go.uber.org/mock/gomock"
) )
@ -72,10 +72,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) DownloadWithContext(ctx, cid, ou
} }
// FetchManifestWithContext mocks base method. // FetchManifestWithContext mocks base method.
func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) { func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error) {
m.ctrl.T.Helper() m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "FetchManifestWithContext", ctx, cid) ret := m.ctrl.Call(m, "FetchManifestWithContext", ctx, cid)
ret0, _ := ret[0].(*communities.CodexManifest) ret0, _ := ret[0].(codex.Manifest)
ret1, _ := ret[1].(error) ret1, _ := ret[1].(error)
return ret0, ret1 return ret0, ret1
} }
@ -156,10 +156,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) SetRequestTimeout(timeout any) *
} }
// TriggerDownload mocks base method. // TriggerDownload mocks base method.
func (m *MockCodexClientInterface) TriggerDownload(cid string) (*communities.CodexManifest, error) { func (m *MockCodexClientInterface) TriggerDownload(cid string) (codex.Manifest, error) {
m.ctrl.T.Helper() m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "TriggerDownload", cid) ret := m.ctrl.Call(m, "TriggerDownload", cid)
ret0, _ := ret[0].(*communities.CodexManifest) ret0, _ := ret[0].(codex.Manifest)
ret1, _ := ret[1].(error) ret1, _ := ret[1].(error)
return ret0, ret1 return ret0, ret1
} }
@ -171,10 +171,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) TriggerDownload(cid any) *gomock
} }
// TriggerDownloadWithContext mocks base method. // TriggerDownloadWithContext mocks base method.
func (m *MockCodexClientInterface) TriggerDownloadWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) { func (m *MockCodexClientInterface) TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error) {
m.ctrl.T.Helper() m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "TriggerDownloadWithContext", ctx, cid) ret := m.ctrl.Call(m, "TriggerDownloadWithContext", ctx, cid)
ret0, _ := ret[0].(*communities.CodexManifest) ret0, _ := ret[0].(codex.Manifest)
ret1, _ := ret[1].(error) ret1, _ := ret[1].(error)
return ret0, ret1 return ret0, ret1
} }

37
communities/testutil.go Normal file
View File

@ -0,0 +1,37 @@
package communities
import (
"testing"
"github.com/codex-storage/codex-go-bindings/codex"
)
func NewCodexClientTest(t *testing.T) *CodexClient {
client, err := NewCodexClient(codex.Config{
DataDir: t.TempDir(),
LogFormat: codex.LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
DiscoveryPort: 8092,
})
if err != nil {
t.Fatalf("Failed to create Codex node: %v", err)
}
err = client.Start()
if err != nil {
t.Fatalf("Failed to start Codex node: %v", err)
}
t.Cleanup(func() {
if err := client.Stop(); err != nil {
t.Logf("cleanup codex: %v", err)
}
if err := client.Destroy(); err != nil {
t.Logf("cleanup codex: %v", err)
}
})
return client
}

3
go.mod
View File

@ -1,6 +1,6 @@
module go-codex-client module go-codex-client
go 1.23.0 go 1.24.0
require ( require (
github.com/stretchr/testify v1.11.1 github.com/stretchr/testify v1.11.1
@ -10,6 +10,7 @@ require (
) )
require ( require (
github.com/codex-storage/codex-go-bindings v0.0.22 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
go.uber.org/multierr v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect

2
go.sum
View File

@ -1,3 +1,5 @@
github.com/codex-storage/codex-go-bindings v0.0.22 h1:53nOqLzgfvR3KdghFAKDoREoW+n12ewvNf8Zf3Pdobc=
github.com/codex-storage/codex-go-bindings v0.0.22/go.mod h1:hP/n9iDZqQP4MytkgUepl3yMMsZy5Jbk9lQbbbVJ51Q=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=