From a74ac8412447ef7c94152b80a459b2e0931b45ae Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 28 Oct 2025 06:56:13 +0100 Subject: [PATCH] Add codex library integration --- .gitignore | 2 + .vscode/settings.json | 11 +- Makefile | 46 ++ README.md | 147 +---- cmd/download/main.go | 29 +- cmd/upload/main.go | 29 +- communities/codex_archive_downloader.go | 4 +- ...dex_archive_downloader_integration_test.go | 23 +- communities/codex_archive_downloader_test.go | 45 +- communities/codex_client.go | 261 ++------ communities/codex_client_integration_test.go | 63 +- communities/codex_client_interface.go | 12 +- communities/codex_client_test.go | 556 ++++-------------- communities/codex_index_downloader.go | 8 +- ...codex_index_downloader_integration_test.go | 13 +- communities/codex_index_downloader_test.go | 43 +- communities/codex_manifest.go | 14 - communities/mock/codex_client_interface.go | 14 +- communities/testutil.go | 37 ++ go.mod | 3 +- go.sum | 2 + 21 files changed, 424 insertions(+), 938 deletions(-) create mode 100644 Makefile delete mode 100644 communities/codex_manifest.go create mode 100644 communities/testutil.go diff --git a/.gitignore b/.gitignore index d8d1dd4..46de977 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ coverage*.cov # Logs *.log + +libs \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 390e9dd..62cfde8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,13 @@ { "go.testTags": "codex_integration", "gopls": { - "buildFlags": ["-tags=codex_integration"] + "buildFlags": [ + "-tags=integration" + ] + }, + "go.toolsEnvVars": { + "CGO_ENABLED": "1", + "CGO_CFLAGS": "-I${workspaceFolder}/libs", + "CGO_LDFLAGS": "-L${workspaceFolder}/libs -lcodex -Wl,-rpath,${workspaceFolder}/libs" } -} +} \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9326732 --- /dev/null +++ b/Makefile @@ -0,0 +1,46 @@ +# Destination folder for the downloaded libraries +LIBS_DIR := $(abspath ./libs) + +# Flags for CGO to find the headers and the shared library +UNAME_S := $(shell uname -s) +CGO_CFLAGS := -I$(LIBS_DIR) +CGO_LDFLAGS := -L$(LIBS_DIR) -lcodex -Wl,-rpath,$(LIBS_DIR) + +ifeq ($(OS),Windows_NT) + BIN_NAME := codex-go.exe +else + BIN_NAME := codex-go +endif + +# Configuration for fetching the right binary +OS ?= "linux" +ARCH ?= "amd64" +VERSION ?= "v0.0.22" +DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(VERSION)/codex-${OS}-${ARCH}.zip" + +fetch: + @echo "Fetching libcodex from GitHub Actions from: ${DOWNLOAD_URL}" + curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${OS}-${ARCH}.zip ${DOWNLOAD_URL} + unzip -o -qq $(LIBS_DIR)/codex-${OS}-${ARCH}.zip -d $(LIBS_DIR) + rm -f $(LIBS_DIR)/*.zip + +build: + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o $(BIN_NAME) main.go + +build-upload: + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-upload ./cmd/upload + +build-download: + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o bin/codex-download ./cmd/download + +test: + @echo "Running unit tests..." + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -v ./communities + +test-integration: + @echo "Running tests..." + CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go test -v -tags=codex_integration ./communities -run Integration -timeout 15s + +clean: + rm -f $(BIN_NAME) + rm -Rf $(LIBS_DIR)/* \ No newline at end of file diff --git a/README.md b/README.md index 96fd27b..97c51e6 100644 --- a/README.md +++ b/README.md @@ -10,25 +10,12 @@ A lightweight Go client utility for interacting with Codex client. We will be running codex client, and then use a small testing utility to check if the low level abstraction - CodexClient - correctly uploads and downloads the content. -### Running CodexClient +### Integration Codex library -I often remove some logging noise, by slightly changing the build -params in `build.nims` (nim-codex): +You need to download the library file by using: -```nim -task codex, "build codex binary": - buildBinary "codex", - # params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - params = - "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE" -``` - -You see a slightly more selective `params` in the `codex` task. - -To run the client I use the following command: - -```bash -./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE +```sh +make fetch ``` ### Building codex-upload and codex-download utilities @@ -36,8 +23,8 @@ To run the client I use the following command: Use the following command to build the `codex-upload` and `codex-download` utilities: ```bash -go build -o bin/codex-upload ./cmd/upload -go build -o bin/codex-download ./cmd/download +make build-upload +make build-download ``` ### Uploading content to Codex @@ -45,8 +32,8 @@ Now, using the `codex-upload` utility, we can upload the content to Codex as fol ```bash ~/code/local/go-codex-client -❯ ./bin/codex-upload -file test-data.bin -host localhost -port 8001 -Uploading test-data.bin (43 bytes) to Codex at localhost:8001... +❯ ./bin/codex-upload -file test-data.bin +Uploading test-data.bin (43 bytes) to Codex ✅ Upload successful! CID: zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V ``` @@ -57,8 +44,8 @@ Now, having the content uploaded to Codex - let's get it back using the `codex-d ```bash ~/code/local/go-codex-client -❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin -host localhost -port 8001 -Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex at localhost:8001... +❯ ./bin/codex-download -cid zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V -file output.bin +Downloading CID zDvZRwzm8K7bcyPeBXcZzWD7AWc4VqNuseduDr3VsuYA1yXej49V from Codex... ✅ Download successful! Saved to: output.bin ``` @@ -85,115 +72,23 @@ next section. To run all unit tests: ```bash -❯ go test -v ./communities -count 1 +❯ make test +=== RUN TestUpload_Success +--- PASS: TestUpload_Success (0.00s) +=== RUN TestDownload_Success +--- PASS: TestDownload_Success (0.00s) +=== RUN TestDownloadWithContext_Cancel +--- PASS: TestDownloadWithContext_Cancel (0.04s) +PASS +ok go-codex-client/communities 0.044s ``` -To be more selective, e.g. in order to run all the tests from -`CodexArchiveDownloaderSuite`, run: +To run the integration test, use `test-integration`: ```bash -go test -v ./communities -run CodexArchiveDownloader -count 1 +make test-integration ``` -or for an individual test from that suite: - -```bash -go test -v ./communities -run TestCodexArchiveDownloaderSuite/TestCancellationDuringPolling -count 1 -``` - -You can also use `gotestsum` to run the tests (you may need to install it first, e.g. `go install gotest.tools/gotestsum@v1.13.0`): - -```bash -gotestsum --packages="./communities" -f testname --rerun-fails -- -count 1 -``` - -For a more verbose output including logs use `-f standard-verbose`, e.g.: - -```bash -gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -v -count 1 -``` - -To be more selective, e.g. in order to run all the tests from -`CodexArchiveDownloaderSuite`, run: - -```bash -gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArchiveDownloader -count 1 -``` - -or for an individual test from that suite: - -```bash -gotestsum --packages="./communities" -f testname --rerun-fails -- -run TestCodexArchiveDownloaderSuite/TestCancellationDuringPolling -count 1 -``` - -Notice, that the `-run` flag accepts a regular expression that matches against the full test path, so you can be more concise in naming if necessary, e.g.: - -```bash -gotestsum --packages="./communities" -f testname --rerun-fails -- -run CodexArchiveDownloader/Cancellation -count 1 -``` - -This also applies to native `go test` command. - -### Running integration tests - -When building Codex client for testing like here, I often remove some logging noise, by slightly changing the build params in `build.nims`: - -```nim -task codex, "build codex binary": - buildBinary "codex", - # params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" - params = - "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:chronicles_enabled_topics:restapi:TRACE,node:TRACE" -``` - -You see a slightly more selective `params` in the `codex` task. - -To start Codex client, use e.g.: - -```bash -./build/codex --data-dir=./data-1 --listen-addrs=/ip4/127.0.0.1/tcp/8081 --api-port=8001 --nat=none --disc-port=8091 --log-level=TRACE -``` - -To run the integration test, use `codex_integration` tag and narrow the scope using `-run Integration`: - -```bash -CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s -``` - -This will run all integration tests, including CodexClient integration tests. - -To make sure that the test is actually run and not cached, use `count` option: - -```bash -CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run Integration -timeout 15s -count 1 -``` - -To be more specific and only run the tests related to, e.g. index downloader or archive -downloader you can use: - -```bash -CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexIndexDownloaderIntegration -timeout 15s -count 1 - -CODEX_API_PORT=8001 go test -v -tags=codex_integration ./communities -run CodexArchiveDownloaderIntegration -timeout 15s -count 1 -``` - -and then, if you prefer to use `gotestsum`: - -```bash -CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexIndexDownloaderIntegration -v -count 1 - -CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -run CodexArchiveDownloaderIntegration -v -count 1 -``` - -or to run all integration tests (including CodexClient integration tests): - -```bash -CODEX_API_PORT=8001 gotestsum --packages="./communities" -f standard-verbose --rerun-fails -- -tags=codex_integration -v -count 1 -run Integration -``` - -I prefer to be more selective when running integration tests. - - ### Regenerating artifacts Everything you need comes included in the repo. But if you decide to change things, diff --git a/cmd/download/main.go b/cmd/download/main.go index 2041efd..d424d6f 100644 --- a/cmd/download/main.go +++ b/cmd/download/main.go @@ -5,14 +5,15 @@ import ( "fmt" "log" "os" + "path" "go-codex-client/communities" // Import the local communities package + + "github.com/codex-storage/codex-go-bindings/codex" ) func main() { var ( - host = flag.String("host", "localhost", "Codex host") - port = flag.String("port", "8080", "Codex port") cid = flag.String("cid", "", "CID of the file to download") file = flag.String("file", "downloaded-file.bin", "File to save the downloaded data") ) @@ -24,7 +25,20 @@ func main() { } // Create Codex client - client := communities.NewCodexClient(*host, *port) + client, err := communities.NewCodexClient(codex.Config{ + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + LogLevel: "ERROR", + DataDir: path.Join(os.TempDir(), "codex-client-data"), + }) + if err != nil { + log.Fatalf("Failed to create CodexClient: %v", err) + } + + if err := client.Start(); err != nil { + log.Fatalf("Failed to start CodexClient: %v", err) + } // Create output file outputFile, err := os.Create(*file) @@ -33,8 +47,6 @@ func main() { } defer outputFile.Close() - fmt.Printf("Downloading CID %s from Codex at %s:%s...\n", *cid, *host, *port) - // Download data - pass the io.Writer (outputFile), not the string err = client.Download(*cid, outputFile) if err != nil { @@ -43,6 +55,13 @@ func main() { log.Fatalf("Download failed: %v", err) } + if err := client.Stop(); err != nil { + log.Printf("Warning: Failed to stop CodexClient: %v", err) + } + if err := client.Destroy(); err != nil { + log.Printf("Warning: Failed to stop CodexClient: %v", err) + } + fmt.Printf("✅ Download successful!\n") fmt.Printf("Saved to: %s\n", *file) } diff --git a/cmd/upload/main.go b/cmd/upload/main.go index db630c2..658f9bf 100644 --- a/cmd/upload/main.go +++ b/cmd/upload/main.go @@ -6,14 +6,15 @@ import ( "fmt" "log" "os" + "path" "go-codex-client/communities" // Import the local communities package + + "github.com/codex-storage/codex-go-bindings/codex" ) func main() { var ( - host = flag.String("host", "localhost", "Codex host") - port = flag.String("port", "8080", "Codex port") file = flag.String("file", "test-data.bin", "File to upload") filename = flag.String("name", "", "Filename to use in upload (defaults to actual filename)") ) @@ -31,15 +32,35 @@ func main() { uploadName = *file } - fmt.Printf("Uploading %s (%d bytes) to Codex at %s:%s...\n", *file, len(data), *host, *port) + fmt.Printf("Uploading %s (%d bytes) to Codex...\n", *file, len(data)) // Create Codex client and upload - client := communities.NewCodexClient(*host, *port) + client, err := communities.NewCodexClient(codex.Config{ + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + LogLevel: "ERROR", + DataDir: path.Join(os.TempDir(), "codex-client-data"), + }) + if err != nil { + log.Fatalf("Failed to create CodexClient: %v", err) + } + + if err := client.Start(); err != nil { + log.Fatalf("Failed to start CodexClient: %v", err) + } cid, err := client.Upload(bytes.NewReader(data), uploadName) if err != nil { log.Fatalf("Upload failed: %v", err) } + if err := client.Stop(); err != nil { + log.Printf("Warning: Failed to stop CodexClient: %v", err) + } + if err := client.Destroy(); err != nil { + log.Printf("Warning: Failed to stop CodexClient: %v", err) + } + fmt.Printf("✅ Upload successful!\n") fmt.Printf("CID: %s\n", cid) } diff --git a/communities/codex_archive_downloader.go b/communities/codex_archive_downloader.go index 84a9b28..5e32b68 100644 --- a/communities/codex_archive_downloader.go +++ b/communities/codex_archive_downloader.go @@ -316,8 +316,8 @@ func (d *CodexArchiveDownloader) triggerSingleArchiveDownload(hash, cid string, return fmt.Errorf("failed to trigger archive download with CID %s: %w", cid, err) } - if manifest.CID != cid { - return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.CID, cid) + if manifest.Cid != cid { + return fmt.Errorf("unexpected manifest CID %s, expected %s", manifest.Cid, cid) } return nil diff --git a/communities/codex_archive_downloader_integration_test.go b/communities/codex_archive_downloader_integration_test.go index 10d2180..e6672b1 100644 --- a/communities/codex_archive_downloader_integration_test.go +++ b/communities/codex_archive_downloader_integration_test.go @@ -8,10 +8,10 @@ import ( "context" "crypto/rand" "encoding/hex" - "os" "testing" "time" + "github.com/codex-storage/codex-go-bindings/codex" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -31,19 +31,18 @@ type CodexArchiveDownloaderIntegrationSuite struct { // SetupSuite runs once before all tests in the suite func (suite *CodexArchiveDownloaderIntegrationSuite) SetupSuite() { - // Use port 8001 as specified by the user - host := communities.GetEnvOrDefault("CODEX_HOST", "localhost") - port := communities.GetEnvOrDefault("CODEX_API_PORT", "8001") - suite.client = communities.NewCodexClient(host, port) - - // Optional request timeout override - if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { - if d, err := time.ParseDuration(ms + "ms"); err == nil { - suite.client.SetRequestTimeout(d) - } + var err error + suite.client, err = communities.NewCodexClient(codex.Config{ + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + LogLevel: "ERROR", + }) + if err != nil { + suite.T().Fatalf("Failed to create CodexClient: %v", err) } - suite.T().Logf("CodexClient configured for %s:%s", host, port) + suite.T().Logf("CodexClient configured for") } // TearDownSuite runs once after all tests in the suite diff --git a/communities/codex_archive_downloader_test.go b/communities/codex_archive_downloader_test.go index 127b4f5..a5de2c5 100644 --- a/communities/codex_archive_downloader_test.go +++ b/communities/codex_archive_downloader_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/codex-storage/codex-go-bindings/codex" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -63,7 +64,7 @@ func (suite *CodexArchiveDownloaderSuite) TestBasicSingleArchive() { // Set up mock expectations - same as before suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). + Return(codex.Manifest{Cid: "test-cid-1"}, nil). Times(1) // First HasCid call returns false, second returns true (simulating polling) @@ -149,7 +150,7 @@ func (suite *CodexArchiveDownloaderSuite) TestMultipleArchives() { for _, cid := range expectedCids { suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), cid). - Return(&communities.CodexManifest{CID: cid}, nil). + Return(codex.Manifest{Cid: cid}, nil). Times(1) // Each archive becomes available after one poll @@ -236,7 +237,7 @@ func (suite *CodexArchiveDownloaderSuite) TestErrorDuringTriggerDownload() { // Mock TriggerDownloadWithContext to simulate an error suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - Return(nil, assert.AnError). // Return a generic error to simulate failure + Return(codex.Manifest{}, assert.AnError). // Return a generic error to simulate failure Times(1) // No HasCid calls should be made since TriggerDownload fails @@ -288,13 +289,13 @@ func (suite *CodexArchiveDownloaderSuite) TestActualCancellationDuringTriggerDow // Use DoAndReturn to create a realistic TriggerDownload that waits for cancellation suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) { // Simulate work by waiting for context cancellation select { case <-time.After(5 * time.Second): // This should never happen in our test - return &communities.CodexManifest{CID: cid}, nil + return codex.Manifest{Cid: cid}, nil case <-ctx.Done(): // Wait for actual context cancellation - return nil, ctx.Err() // Return the actual cancellation error + return codex.Manifest{}, ctx.Err() // Return the actual cancellation error } }). Times(1) @@ -352,7 +353,7 @@ func (suite *CodexArchiveDownloaderSuite) TestCancellationDuringPolling() { // Mock successful TriggerDownload suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). + Return(codex.Manifest{Cid: "test-cid-1"}, nil). Times(1) // Mock polling - allow multiple calls, but we'll cancel before completion @@ -420,7 +421,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPollingTimeout() { // Mock successful TriggerDownload suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "test-cid-1"). - Return(&communities.CodexManifest{CID: "test-cid-1"}, nil). + Return(codex.Manifest{Cid: "test-cid-1"}, nil). Times(1) // Mock polling to always return false (simulating timeout) @@ -496,7 +497,7 @@ func (suite *CodexArchiveDownloaderSuite) TestWithExistingArchives() { // Only archive-2 should be downloaded (not in existingArchiveIDs) suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(&communities.CodexManifest{CID: "cid-2"}, nil). + Return(codex.Manifest{Cid: "cid-2"}, nil). Times(1) // Only one call expected // Only archive-2 should be polled @@ -577,7 +578,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_OneSuccessOneError( // Archive-2 succeeds suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(&communities.CodexManifest{CID: "cid-2"}, nil) + Return(codex.Manifest{Cid: "cid-2"}, nil) suite.mockClient.EXPECT(). HasCid("cid-2"). Return(true, nil) @@ -585,7 +586,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_OneSuccessOneError( // Archive-1 fails suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-1"). - Return(nil, fmt.Errorf("trigger failed")) + Return(codex.Manifest{}, fmt.Errorf("trigger failed")) logger := zap.NewNop() downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger) @@ -633,7 +634,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessErrorCancell // Archive-3 (newest) succeeds suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-3"). - Return(&communities.CodexManifest{CID: "cid-3"}, nil) + Return(codex.Manifest{Cid: "cid-3"}, nil) suite.mockClient.EXPECT(). HasCid("cid-3"). Return(true, nil) @@ -641,14 +642,14 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessErrorCancell // Archive-2 fails suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(nil, fmt.Errorf("trigger failed")) + Return(codex.Manifest{}, fmt.Errorf("trigger failed")) // Archive-1 will be cancelled (no expectations needed) suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-1"). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) { <-ctx.Done() // Wait for cancellation - return nil, ctx.Err() + return codex.Manifest{}, ctx.Err() }). AnyTimes() @@ -700,7 +701,7 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessThenCancella // Archive-2 (newer) succeeds suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(&communities.CodexManifest{CID: "cid-2"}, nil) + Return(codex.Manifest{Cid: "cid-2"}, nil) suite.mockClient.EXPECT(). HasCid("cid-2"). Return(true, nil) @@ -708,9 +709,9 @@ func (suite *CodexArchiveDownloaderSuite) TestPartialSuccess_SuccessThenCancella // Archive-1 will be cancelled suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-1"). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) { <-ctx.Done() // Wait for cancellation - return nil, ctx.Err() + return codex.Manifest{}, ctx.Err() }). AnyTimes() @@ -762,9 +763,9 @@ func (suite *CodexArchiveDownloaderSuite) TestNoSuccess_OnlyCancellation() { // Both archives will be cancelled suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) { <-ctx.Done() // Wait for cancellation - return nil, ctx.Err() + return codex.Manifest{}, ctx.Err() }). AnyTimes() @@ -815,10 +816,10 @@ func (suite *CodexArchiveDownloaderSuite) TestNoSuccess_OnlyErrors() { // Both archives fail suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-1"). - Return(nil, fmt.Errorf("trigger failed for cid-1")) + Return(codex.Manifest{}, fmt.Errorf("trigger failed for cid-1")) suite.mockClient.EXPECT(). TriggerDownloadWithContext(gomock.Any(), "cid-2"). - Return(nil, fmt.Errorf("trigger failed for cid-2")) + Return(codex.Manifest{}, fmt.Errorf("trigger failed for cid-2")) logger := zap.NewNop() downloader := communities.NewCodexArchiveDownloader(suite.mockClient, index, communityID, []string{}, cancelChan, logger) diff --git a/communities/codex_client.go b/communities/codex_client.go index ea07276..357c25a 100644 --- a/communities/codex_client.go +++ b/communities/codex_client.go @@ -9,63 +9,48 @@ package communities import ( "bytes" "context" - "encoding/json" "fmt" "io" - "net/http" - "strings" - "time" + + "github.com/codex-storage/codex-go-bindings/codex" ) // CodexClient handles basic upload/download operations with Codex storage type CodexClient struct { - BaseURL string - Client *http.Client + node *codex.CodexNode + config *codex.Config } // NewCodexClient creates a new Codex client -func NewCodexClient(host string, port string) *CodexClient { - return &CodexClient{ - BaseURL: fmt.Sprintf("http://%s:%s", host, port), - Client: &http.Client{Timeout: 60 * time.Second}, +func NewCodexClient(config codex.Config) (*CodexClient, error) { + node, err := codex.New(config) + if err != nil { + return nil, fmt.Errorf("failed to create Codex node: %w", err) } + + return &CodexClient{ + node: node, + config: &config, + }, nil +} + +func (c CodexClient) Start() error { + return c.node.Start() +} + +func (c CodexClient) Stop() error { + return c.node.Stop() +} + +func (c CodexClient) Destroy() error { + return c.node.Destroy() } // Upload uploads data from a reader to Codex and returns the CID func (c *CodexClient) Upload(data io.Reader, filename string) (string, error) { - url := fmt.Sprintf("%s/api/codex/v1/data", c.BaseURL) - - // Create the HTTP request - req, err := http.NewRequest("POST", url, data) - if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) - } - - // Set headers - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("Content-Disposition", fmt.Sprintf(`filename="%s"`, filename)) - - // Send request - resp, err := c.Client.Do(req) - if err != nil { - return "", fmt.Errorf("failed to upload to codex: %w", err) - } - defer resp.Body.Close() - - // Check if request was successful - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("codex upload failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Read the CID response - cidBytes, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read response: %w", err) - } - - cid := strings.TrimSpace(string(cidBytes)) - return cid, nil + return c.node.UploadReader(codex.UploadOptions{ + Filepath: filename, + }, data) } // Download downloads data from Codex by CID and writes it to the provided writer @@ -73,201 +58,43 @@ func (c *CodexClient) Download(cid string, output io.Writer) error { return c.DownloadWithContext(context.Background(), cid, output) } -func (c *CodexClient) TriggerDownload(cid string) (*CodexManifest, error) { +func (c *CodexClient) TriggerDownload(cid string) (codex.Manifest, error) { return c.TriggerDownloadWithContext(context.Background(), cid) } func (c *CodexClient) HasCid(cid string) (bool, error) { - url := fmt.Sprintf("%s/api/codex/v1/data/%s/exists", c.BaseURL, cid) - - resp, err := c.Client.Get(url) - if err != nil { - return false, fmt.Errorf("failed to check cid existence: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return false, fmt.Errorf("cid check failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Parse JSON response: {"": } - var result map[string]bool - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return false, fmt.Errorf("failed to parse response: %w", err) - } - - // Validate the CID key matches request - hasCid, exists := result[cid] - if !exists { - return false, fmt.Errorf("response missing CID key %q", cid) - } - - return hasCid, nil + err := c.LocalDownload(cid, io.Discard) + return err == nil, nil } func (c *CodexClient) RemoveCid(cid string) error { - url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid) - - req, err := http.NewRequest("DELETE", url, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return fmt.Errorf("failed trying to delete cid: %s, %w", cid, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNoContent { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("cid delete failed with status %d: %s", resp.StatusCode, string(body)) - } - - return nil + return c.node.Delete(cid) } // DownloadWithContext downloads data from Codex by CID with cancellation support func (c *CodexClient) DownloadWithContext(ctx context.Context, cid string, output io.Writer) error { - url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/stream", c.BaseURL, cid) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return fmt.Errorf("failed to download from codex: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Use context-aware copy for cancellable streaming - return c.copyWithContext(ctx, output, resp.Body) + return c.node.DownloadStream(cid, codex.DownloadStreamOptions{ + Writer: output, + }) } func (c *CodexClient) LocalDownload(cid string, output io.Writer) error { - return c.LocalDownloadWithContext(context.Background(), cid, output) + return c.node.DownloadStream(cid, codex.DownloadStreamOptions{ + Writer: output, + Local: true, + }) } func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error { - url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return fmt.Errorf("failed to download from codex: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Use context-aware copy for cancellable streaming - return c.copyWithContext(ctx, output, resp.Body) + return c.LocalDownload(cid, output) } -func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) { - url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/manifest", c.BaseURL, cid) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to fetch manifest from codex: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("codex fetch manifest failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Parse JSON response containing manifest - var manifest CodexManifest - if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { - return nil, fmt.Errorf("failed to parse manifest: %w", err) - } - - return &manifest, nil +func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error) { + return c.node.DownloadManifest(cid) } -func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) { - url := fmt.Sprintf("%s/api/codex/v1/data/%s/network", c.BaseURL, cid) - - req, err := http.NewRequestWithContext(ctx, "POST", url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - - resp, err := c.Client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to trigger download from codex: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("codex async download failed with status %d: %s", resp.StatusCode, string(body)) - } - - // Parse JSON response containing manifest - var manifest CodexManifest - if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { - return nil, fmt.Errorf("failed to parse download manifest: %w", err) - } - - return &manifest, nil -} - -// copyWithContext performs io.Copy but respects context cancellation -func (c *CodexClient) copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { - // Create a buffer for chunked copying - buf := make([]byte, 64*1024) // 64KB buffer - - for { - select { - case <-ctx.Done(): - return ctx.Err() // Return cancellation error - default: - } - - // Read a chunk - n, err := src.Read(buf) - if n > 0 { - // Write the chunk - if _, writeErr := dst.Write(buf[:n]); writeErr != nil { - return fmt.Errorf("failed to write data: %w", writeErr) - } - } - - if err == io.EOF { - return nil // Successful completion - } - if err != nil { - return fmt.Errorf("failed to read data: %w", err) - } - } -} - -// SetRequestTimeout sets the HTTP client timeout for requests -func (c *CodexClient) SetRequestTimeout(timeout time.Duration) { - c.Client.Timeout = timeout +func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error) { + return c.node.Fetch(cid) } // UploadArchive is a convenience method for uploading archive data diff --git a/communities/codex_client_integration_test.go b/communities/codex_client_integration_test.go index 0917dac..902a22d 100644 --- a/communities/codex_client_integration_test.go +++ b/communities/codex_client_integration_test.go @@ -8,10 +8,10 @@ import ( "context" "crypto/rand" "encoding/hex" - "os" "testing" "time" + "github.com/codex-storage/codex-go-bindings/codex" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -20,29 +20,22 @@ import ( ) // CodexClientIntegrationTestSuite demonstrates testify's suite functionality for CodexClient integration tests -// These tests exercise real network calls against a running Codex node. -// Required env vars (with defaults): -// - CODEX_HOST (default: localhost) -// - CODEX_API_PORT (default: 8080) -// - CODEX_TIMEOUT_MS (optional; default: 60000) type CodexClientIntegrationTestSuite struct { suite.Suite client *communities.CodexClient - host string - port string } // SetupSuite runs once before all tests in the suite func (suite *CodexClientIntegrationTestSuite) SetupSuite() { - suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost") - suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8080") - suite.client = communities.NewCodexClient(suite.host, suite.port) - - // Optional request timeout override - if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { - if d, err := time.ParseDuration(ms + "ms"); err == nil { - suite.client.SetRequestTimeout(d) - } + var err error + suite.client, err = communities.NewCodexClient(codex.Config{ + DataDir: suite.T().TempDir(), + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + }) + if err != nil { + suite.T().Fatalf("Failed to create Codex client: %v", err) } } @@ -114,15 +107,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_CheckNonExistingCI } func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload() { - // Use port 8001 for this test as specified - client := communities.NewCodexClient(suite.host, "8001") - - // Optional request timeout override - if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { - if d, err := time.ParseDuration(ms + "ms"); err == nil { - client.SetRequestTimeout(d) - } - } + client := communities.NewCodexClientTest(suite.T()) // Generate random payload to ensure proper round-trip verification payload := make([]byte, 1024) @@ -145,7 +130,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload() // Trigger async download manifest, err := client.TriggerDownload(cid) require.NoError(suite.T(), err, "TriggerDownload failed") - suite.T().Logf("Async download triggered, manifest CID: %s", manifest.CID) + suite.T().Logf("Async download triggered, manifest CID: %s", manifest.Cid) // Poll HasCid for up to 10 seconds using goroutine and channel downloadComplete := make(chan bool, 1) @@ -221,27 +206,27 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_FetchManifest() { manifest, err := suite.client.FetchManifestWithContext(ctx, cid) require.NoError(suite.T(), err, "FetchManifestWithContext failed") - suite.T().Logf("FetchManifest successful, manifest CID: %s", manifest.CID) + suite.T().Logf("FetchManifest successful, manifest CID: %s", manifest.Cid) // Verify manifest properties - assert.Equal(suite.T(), cid, manifest.CID, "Manifest CID mismatch") + assert.Equal(suite.T(), cid, manifest.Cid, "Manifest CID mismatch") // Verify manifest has expected fields - assert.NotEmpty(suite.T(), manifest.Manifest.TreeCid, "Expected TreeCid to be non-empty") - suite.T().Logf("Manifest TreeCid: %s", manifest.Manifest.TreeCid) + assert.NotEmpty(suite.T(), manifest.TreeCid, "Expected TreeCid to be non-empty") + suite.T().Logf("Manifest TreeCid: %s", manifest.TreeCid) - assert.Greater(suite.T(), manifest.Manifest.DatasetSize, int64(0), "Expected DatasetSize > 0") - suite.T().Logf("Manifest DatasetSize: %d", manifest.Manifest.DatasetSize) + assert.Greater(suite.T(), manifest.DatasetSize, 0, "Expected DatasetSize > 0") + suite.T().Logf("Manifest DatasetSize: %d", manifest.DatasetSize) - assert.Greater(suite.T(), manifest.Manifest.BlockSize, 0, "Expected BlockSize > 0") - suite.T().Logf("Manifest BlockSize: %d", manifest.Manifest.BlockSize) + assert.Greater(suite.T(), manifest.BlockSize, 0, "Expected BlockSize > 0") + suite.T().Logf("Manifest BlockSize: %d", manifest.BlockSize) - assert.Equal(suite.T(), "fetch-manifest-test.bin", manifest.Manifest.Filename, "Filename mismatch") - suite.T().Logf("Manifest Filename: %s", manifest.Manifest.Filename) + assert.Equal(suite.T(), "fetch-manifest-test.bin", manifest.Filename, "Filename mismatch") + suite.T().Logf("Manifest Filename: %s", manifest.Filename) // Log manifest details for verification - suite.T().Logf("Manifest Protected: %v", manifest.Manifest.Protected) - suite.T().Logf("Manifest Mimetype: %s", manifest.Manifest.Mimetype) + suite.T().Logf("Manifest Protected: %v", manifest.Protected) + suite.T().Logf("Manifest Mimetype: %s", manifest.Mimetype) // Test fetching manifest for non-existent CID (should fail gracefully) nonExistentCID := "zDvZRwzmNonExistentCID123456789" diff --git a/communities/codex_client_interface.go b/communities/codex_client_interface.go index b2afee2..ec05ea8 100644 --- a/communities/codex_client_interface.go +++ b/communities/codex_client_interface.go @@ -3,7 +3,8 @@ package communities import ( "context" "io" - "time" + + "github.com/codex-storage/codex-go-bindings/codex" ) // Mock generation instruction above will create a mock in package `mock_communities` @@ -25,16 +26,13 @@ type CodexClientInterface interface { LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error // Async download methods - TriggerDownload(cid string) (*CodexManifest, error) - TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) + TriggerDownload(cid string) (codex.Manifest, error) + TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error) // Manifest methods - FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) + FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error) // CID management methods HasCid(cid string) (bool, error) RemoveCid(cid string) error - - // Configuration methods - SetRequestTimeout(timeout time.Duration) } diff --git a/communities/codex_client_test.go b/communities/codex_client_test.go index 10f1f90..cf4fbc2 100644 --- a/communities/codex_client_test.go +++ b/communities/codex_client_test.go @@ -4,10 +4,7 @@ import ( "bytes" "context" "errors" - "fmt" "io" - "net/http" - "net/http/httptest" "testing" "time" @@ -18,24 +15,33 @@ import ( "go-codex-client/communities" ) +func upload(client communities.CodexClient, t *testing.T, buf *bytes.Buffer) string { + filename := "hello.txt" + cid, err := client.Upload(buf, filename) + if err != nil { + t.Fatalf("Failed to upload file: %v", err) + } + + if cid == "" { + t.Fatalf("Expected non-empty CID after upload") + } + + return cid +} + // CodexClientTestSuite demonstrates testify's suite functionality for CodexClient tests type CodexClientTestSuite struct { suite.Suite client *communities.CodexClient - server *httptest.Server } // SetupTest runs before each test method func (suite *CodexClientTestSuite) SetupTest() { - suite.client = communities.NewCodexClient("localhost", "8080") + suite.client = communities.NewCodexClientTest(suite.T()) } // TearDownTest runs after each test method func (suite *CodexClientTestSuite) TearDownTest() { - if suite.server != nil { - suite.server.Close() - suite.server = nil - } } // TestCodexClientTestSuite runs the test suite @@ -44,110 +50,33 @@ func TestCodexClientTestSuite(t *testing.T) { } func (suite *CodexClientTestSuite) TestUpload_Success() { - // Arrange a fake Codex server that validates headers and returns a CID - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - if r.URL.Path != "/api/codex/v1/data" { - w.WriteHeader(http.StatusNotFound) - return - } - - if ct := r.Header.Get("Content-Type"); ct != "application/octet-stream" { - w.WriteHeader(http.StatusBadRequest) - return - } - if cd := r.Header.Get("Content-Disposition"); cd != "filename=\"hello.txt\"" { - w.WriteHeader(http.StatusBadRequest) - return - } - - _, _ = io.ReadAll(r.Body) // consume body - _ = r.Body.Close() - - w.WriteHeader(http.StatusOK) - // Codex returns CIDv1 base58btc - // prefix: zDv - // - z = multibase prefix for base58btc - // - Dv = CIDv1 prefix for raw codex - // we add a newline to simulate real response - _, _ = w.Write([]byte("zDvZRwzmTestCID123\n")) - })) - - suite.client.BaseURL = suite.server.URL - // Act cid, err := suite.client.Upload(bytes.NewReader([]byte("payload")), "hello.txt") // Assert require.NoError(suite.T(), err) // Codex uses CIDv1 with base58btc encoding (prefix: zDv) - assert.Equal(suite.T(), "zDvZRwzmTestCID123", cid) -} - -func (suite *CodexClientTestSuite) TestDownload_Success() { - const wantCID = "zDvZRwzm" - const payload = "hello from codex" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - if r.URL.Path != "/api/codex/v1/data/"+wantCID+"/network/stream" { - w.WriteHeader(http.StatusNotFound) - return - } - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(payload)) - })) - - suite.client.BaseURL = suite.server.URL - - var buf bytes.Buffer - err := suite.client.Download(wantCID, &buf) - require.NoError(suite.T(), err) - assert.Equal(suite.T(), payload, buf.String()) + assert.Equal(suite.T(), "zDvZRwzmBEaJ338xaCHbKbGAJ4X41YyccS6eyorrYBbmPnWuLxCh", cid) } func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() { - const cid = "zDvZRwzm" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/codex/v1/data/"+cid+"/network/stream" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/octet-stream") - flusher, _ := w.(http.Flusher) - w.WriteHeader(http.StatusOK) - // Stream data slowly so the request can be canceled - for i := 0; i < 1000; i++ { - select { - case <-r.Context().Done(): - return - default: - } - if _, err := w.Write([]byte("x")); err != nil { - // Client likely went away; stop writing - return - } - if flusher != nil { - flusher.Flush() - } - time.Sleep(10 * time.Millisecond) - } - })) - - suite.client.BaseURL = suite.server.URL + // skip test + suite.T().Skip("Wait for cancellation support PR to be merged in codex-go-bindings") + len := 1024 * 1024 * 50 + buf := bytes.NewBuffer(make([]byte, len)) + cid := upload(*suite.client, suite.T(), buf) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) - defer cancel() - err := suite.client.DownloadWithContext(ctx, cid, io.Discard) + channelError := make(chan error, 1) + go func() { + err := suite.client.DownloadWithContext(ctx, cid, io.Discard) + channelError <- err + }() + + cancel() + err := <-channelError + require.Error(suite.T(), err) // Accept either canceled or deadline exceeded depending on timing if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { @@ -160,31 +89,20 @@ func (suite *CodexClientTestSuite) TestDownloadWithContext_Cancel() { } func (suite *CodexClientTestSuite) TestHasCid_Success() { + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) + tests := []struct { name string cid string - hasIt bool wantBool bool }{ - {"has CID returns true", "zDvZRwzmTestCID", true, true}, - {"has CID returns false", "zDvZRwzmTestCID", false, false}, + {"has CID returns true", cid, true}, + {"has CID returns false", "zDvZRwzmBEaJ338xaCHbKbGAJ4X41YyccS6eyorrYBbmPnWuLxCe", false}, } for _, tt := range tests { suite.Run(tt.name, func() { - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/codex/v1/data/"+tt.cid+"/exists" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - // Return JSON: {"": } - fmt.Fprintf(w, `{"%s": %t}`, tt.cid, tt.hasIt) - })) - - suite.client.BaseURL = suite.server.URL - got, err := suite.client.HasCid(tt.cid) require.NoError(suite.T(), err) assert.Equal(suite.T(), tt.wantBool, got, "HasCid(%q) = %v, want %v", tt.cid, got, tt.wantBool) @@ -192,181 +110,42 @@ func (suite *CodexClientTestSuite) TestHasCid_Success() { } } -func (suite *CodexClientTestSuite) TestHasCid_RequestError() { - // Create a server and immediately close it to trigger connection error - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - suite.server.Close() // Close immediately so connection fails +func (suite *CodexClientTestSuite) TestDownload_Success() { + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) - suite.client.BaseURL = suite.server.URL // Use the closed server's URL - - got, err := suite.client.HasCid("zDvZRwzmTestCID") - require.Error(suite.T(), err) - assert.False(suite.T(), got, "expected false on error") -} - -func (suite *CodexClientTestSuite) TestHasCid_CidMismatch() { - const requestCid = "zDvZRwzmRequestCID" - const responseCid = "zDvZRwzmDifferentCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - // Return a different CID in the response - fmt.Fprintf(w, `{"%s": true}`, responseCid) - })) - - suite.client.BaseURL = suite.server.URL - - got, err := suite.client.HasCid(requestCid) - require.Error(suite.T(), err, "expected error for CID mismatch") - assert.False(suite.T(), got, "expected false on CID mismatch") - // Check error message mentions the missing/mismatched CID - assert.Contains(suite.T(), err.Error(), requestCid, "error should mention request CID") + var buf bytes.Buffer + err := suite.client.Download(cid, &buf) + require.NoError(suite.T(), err) + assert.Equal(suite.T(), payload, buf.String()) } func (suite *CodexClientTestSuite) TestRemoveCid_Success() { - const testCid = "zDvZRwzmTestCID" + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodDelete { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - if r.URL.Path != "/api/codex/v1/data/"+testCid { - w.WriteHeader(http.StatusNotFound) - return - } - // DELETE should return 204 No Content - w.WriteHeader(http.StatusNoContent) - })) - - suite.client.BaseURL = suite.server.URL - - err := suite.client.RemoveCid(testCid) + err := suite.client.RemoveCid(cid) require.NoError(suite.T(), err) } -func (suite *CodexClientTestSuite) TestRemoveCid_Error() { - const testCid = "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Return error status - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("server error")) - })) - - suite.client.BaseURL = suite.server.URL - - err := suite.client.RemoveCid(testCid) - require.Error(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "500", "error should mention status 500") -} - func (suite *CodexClientTestSuite) TestTriggerDownload() { - const testCid = "zDvZRwzmTestCID" - const expectedManifest = `{ - "cid": "zDvZRwzmTestCID", - "manifest": { - "treeCid": "zDvZRwzmTreeCID", - "datasetSize": 1024, - "blockSize": 65536, - "protected": false, - "filename": "test-file.bin", - "mimetype": "application/octet-stream" - } - }` - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - if r.URL.Path != "/api/codex/v1/data/"+testCid+"/network" { - w.WriteHeader(http.StatusNotFound) - return - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(expectedManifest)) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) ctx := context.Background() - manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid) + manifest, err := suite.client.TriggerDownloadWithContext(ctx, cid) require.NoError(suite.T(), err) - assert.Equal(suite.T(), testCid, manifest.CID) - assert.Equal(suite.T(), "zDvZRwzmTreeCID", manifest.Manifest.TreeCid) - assert.Equal(suite.T(), int64(1024), manifest.Manifest.DatasetSize) - assert.Equal(suite.T(), "test-file.bin", manifest.Manifest.Filename) -} - -func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_RequestError() { - // Create a server and immediately close it to trigger connection error - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - suite.server.Close() - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.TriggerDownloadWithContext(ctx, "zDvZRwzmRigWseNB7WqmudkKAPgZmrDCE9u5cY4KvCqhRo9Ki") - require.Error(suite.T(), err) - assert.Nil(suite.T(), manifest, "expected nil manifest on error") -} - -func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_JSONParseError() { - const testCid = "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - // Return invalid JSON - w.Write([]byte(`{"invalid": json}`)) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid) - require.Error(suite.T(), err, "expected JSON parse error") - assert.Nil(suite.T(), manifest, "expected nil manifest on parse error") - assert.Contains(suite.T(), err.Error(), "failed to parse download manifest", "error should mention parse failure") -} - -func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_HTTPError() { - const testCid = "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("CID not found")) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.TriggerDownloadWithContext(ctx, testCid) - require.Error(suite.T(), err, "expected error for 404 status") - assert.Nil(suite.T(), manifest, "expected nil manifest on HTTP error") - assert.Contains(suite.T(), err.Error(), "404", "error should mention status 404") + assert.Equal(suite.T(), cid, manifest.Cid) + assert.Equal(suite.T(), "zDzSvJTf7mGkC3yuiVGco7Qc6s4LA8edye9inT4w2QqHnfbuRvMr", manifest.TreeCid) + assert.Equal(suite.T(), len(payload), manifest.DatasetSize) + assert.Equal(suite.T(), "hello.txt", manifest.Filename) } func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation() { + suite.T().Skip("Not sure if we are going to have cancellation in trigger download") + const testCid = "zDvZRwzmTestCID" - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Simulate slow response to allow cancellation - select { - case <-r.Context().Done(): - return - case <-time.After(200 * time.Millisecond): - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"cid": "test"}`)) - } - })) - - suite.client.BaseURL = suite.server.URL - // Cancel after 50ms (before server responds) ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() @@ -385,99 +164,43 @@ func (suite *CodexClientTestSuite) TestTriggerDownloadWithContext_Cancellation() } func (suite *CodexClientTestSuite) TestLocalDownload() { - testData := []byte("test data for local download") - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Verify request method and path - assert.Equal(suite.T(), "GET", r.Method, "Expected GET request") - expectedPath := "/api/codex/v1/data/" + testCid - assert.Equal(suite.T(), expectedPath, r.URL.Path, "Expected correct path") - - w.WriteHeader(http.StatusOK) - w.Write(testData) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "test data for local download" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) var buf bytes.Buffer - err := suite.client.LocalDownload(testCid, &buf) + err := suite.client.LocalDownload(cid, &buf) require.NoError(suite.T(), err, "LocalDownload failed") - assert.Equal(suite.T(), testData, buf.Bytes(), "Downloaded data mismatch") + assert.Equal(suite.T(), payload, buf.String(), "Downloaded data mismatch") } func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Success() { - testData := []byte("test data for local download with context") - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Verify request method and path - assert.Equal(suite.T(), "GET", r.Method, "Expected GET request") - expectedPath := "/api/codex/v1/data/" + testCid - assert.Equal(suite.T(), expectedPath, r.URL.Path, "Expected correct path") - - w.WriteHeader(http.StatusOK) - w.Write(testData) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "test data for local download with context" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) ctx := context.Background() var buf bytes.Buffer - err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf) + err := suite.client.LocalDownloadWithContext(ctx, cid, &buf) require.NoError(suite.T(), err, "LocalDownloadWithContext failed") - assert.Equal(suite.T(), testData, buf.Bytes(), "Downloaded data mismatch") -} - -func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_RequestError() { - // Create a server and immediately close it to trigger connection error - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - suite.server.Close() - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - var buf bytes.Buffer - err := suite.client.LocalDownloadWithContext(ctx, "zDvZRwzmTestCID", &buf) - require.Error(suite.T(), err, "Expected error due to closed server") - assert.Contains(suite.T(), err.Error(), "failed to download from codex") -} - -func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_HTTPError() { - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("CID not found in local storage")) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - var buf bytes.Buffer - err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf) - require.Error(suite.T(), err, "Expected error for HTTP 404") - assert.Contains(suite.T(), err.Error(), "404", "Expected '404' in error message") + assert.Equal(suite.T(), payload, buf.String(), "Downloaded data mismatch") } func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() { - testCid := "zDvZRwzmTestCID" + suite.T().Skip("Wait for cancellation support PR to be merged in codex-go-bindings") - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Simulate a slow response - time.Sleep(100 * time.Millisecond) - w.WriteHeader(http.StatusOK) - w.Write([]byte("slow response")) - })) - - suite.client.BaseURL = suite.server.URL - - // Create a context with a very short timeout + len := 1024 * 1024 * 50 + buf := bytes.NewBuffer(make([]byte, len)) + cid := upload(*suite.client, suite.T(), buf) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - defer cancel() - var buf bytes.Buffer - err := suite.client.LocalDownloadWithContext(ctx, testCid, &buf) + channelError := make(chan error, 1) + go func() { + err := suite.client.LocalDownloadWithContext(ctx, cid, io.Discard) + channelError <- err + }() + + cancel() + err := <-channelError + require.Error(suite.T(), err, "Expected context cancellation error") // Accept either canceled or deadline exceeded depending on timing if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { @@ -490,107 +213,28 @@ func (suite *CodexClientTestSuite) TestLocalDownloadWithContext_Cancellation() { } func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Success() { - testCid := "zDvZRwzmTestCID" - expectedManifest := `{ - "cid": "zDvZRwzmTestCID", - "manifest": { - "treeCid": "zDvZRwzmTreeCID123", - "datasetSize": 1024, - "blockSize": 256, - "protected": true, - "filename": "test-file.bin", - "mimetype": "application/octet-stream" - } - }` - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(suite.T(), http.MethodGet, r.Method) - expectedPath := fmt.Sprintf("/api/codex/v1/data/%s/network/manifest", testCid) - assert.Equal(suite.T(), expectedPath, r.URL.Path) - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(expectedManifest)) - })) - - suite.client.BaseURL = suite.server.URL + const payload = "hello from codex" + cid := upload(*suite.client, suite.T(), bytes.NewBuffer([]byte(payload))) ctx := context.Background() - manifest, err := suite.client.FetchManifestWithContext(ctx, testCid) + manifest, err := suite.client.FetchManifestWithContext(ctx, cid) require.NoError(suite.T(), err, "Expected no error") require.NotNil(suite.T(), manifest, "Expected manifest, got nil") - assert.Equal(suite.T(), testCid, manifest.CID) - assert.Equal(suite.T(), "zDvZRwzmTreeCID123", manifest.Manifest.TreeCid) - assert.Equal(suite.T(), int64(1024), manifest.Manifest.DatasetSize) - assert.Equal(suite.T(), 256, manifest.Manifest.BlockSize) - assert.True(suite.T(), manifest.Manifest.Protected, "Expected Protected to be true") - assert.Equal(suite.T(), "test-file.bin", manifest.Manifest.Filename) - assert.Equal(suite.T(), "application/octet-stream", manifest.Manifest.Mimetype) -} - -func (suite *CodexClientTestSuite) TestFetchManifestWithContext_RequestError() { - // Create a server and immediately close it to trigger connection error - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - suite.server.Close() - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.FetchManifestWithContext(ctx, "test-cid") - require.Error(suite.T(), err, "Expected error for closed server") - assert.Nil(suite.T(), manifest, "Expected nil manifest on error") - assert.Contains(suite.T(), err.Error(), "failed to fetch manifest from codex") -} - -func (suite *CodexClientTestSuite) TestFetchManifestWithContext_HTTPError() { - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - w.Write([]byte("Manifest not found")) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.FetchManifestWithContext(ctx, testCid) - require.Error(suite.T(), err, "Expected error for HTTP 404") - assert.Nil(suite.T(), manifest, "Expected nil manifest on error") - assert.Contains(suite.T(), err.Error(), "404", "Expected '404' in error message") -} - -func (suite *CodexClientTestSuite) TestFetchManifestWithContext_JSONParseError() { - testCid := "zDvZRwzmTestCID" - - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte("invalid json {")) - })) - - suite.client.BaseURL = suite.server.URL - - ctx := context.Background() - manifest, err := suite.client.FetchManifestWithContext(ctx, testCid) - require.Error(suite.T(), err, "Expected error for invalid JSON") - assert.Nil(suite.T(), manifest, "Expected nil manifest on JSON parse error") - assert.Contains(suite.T(), err.Error(), "failed to parse manifest", "Expected 'failed to parse manifest' in error message") + assert.Equal(suite.T(), cid, manifest.Cid) + assert.Equal(suite.T(), "zDzSvJTf7mGkC3yuiVGco7Qc6s4LA8edye9inT4w2QqHnfbuRvMr", manifest.TreeCid) + assert.Equal(suite.T(), len(payload), manifest.DatasetSize) + assert.Equal(suite.T(), 65536, manifest.BlockSize) + assert.True(suite.T(), !manifest.Protected, "Expected Protected to be false") + assert.Equal(suite.T(), "hello.txt", manifest.Filename) + assert.Equal(suite.T(), "text/plain", manifest.Mimetype) } func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() { + suite.T().Skip("Not sure if we are going to have cancellation in fetch manifest") + testCid := "zDvZRwzmTestCID" - suite.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Simulate a slow response - time.Sleep(100 * time.Millisecond) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"cid": "test"}`)) - })) - - suite.client.BaseURL = suite.server.URL - // Create a context with a very short timeout ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer cancel() @@ -607,4 +251,30 @@ func (suite *CodexClientTestSuite) TestFetchManifestWithContext_Cancellation() { suite.T().Fatalf("expected context cancellation, got: %v", err) } } + + buf := bytes.NewBuffer([]byte("Hello World!")) + if buf.Len() != manifest.DatasetSize { + suite.T().Errorf("expected size %d, got %d", buf.Len(), manifest.DatasetSize) + } + + defaultBlockSize := 1024 * 64 + if manifest.BlockSize != defaultBlockSize { + suite.T().Errorf("expected block size %d, got %d", defaultBlockSize, manifest.BlockSize) + } + + if manifest.Filename != "test.txt" { + suite.T().Errorf("expected filename %q, got %q", "test.txt", manifest.Filename) + } + + if manifest.Protected { + suite.T().Errorf("expected protected to be false, got true") + } + + if manifest.Mimetype != "text/plain" { + suite.T().Errorf("expected mimetype %q, got %q", "text/plain", manifest.Mimetype) + } + + if manifest.TreeCid == "" { + suite.T().Errorf("expected non-empty TreeCid") + } } diff --git a/communities/codex_index_downloader.go b/communities/codex_index_downloader.go index fbe5333..9f4ebf6 100644 --- a/communities/codex_index_downloader.go +++ b/communities/codex_index_downloader.go @@ -79,19 +79,19 @@ func (d *CodexIndexDownloader) GotManifest() <-chan struct{} { } // Verify that the CID matches our configured indexCid - if manifest.CID != d.indexCid { + if manifest.Cid != d.indexCid { d.mu.Lock() - d.downloadError = fmt.Errorf("manifest CID mismatch: expected %s, got %s", d.indexCid, manifest.CID) + d.downloadError = fmt.Errorf("manifest CID mismatch: expected %s, got %s", d.indexCid, manifest.Cid) d.mu.Unlock() d.logger.Debug("manifest CID mismatch", zap.String("expected", d.indexCid), - zap.String("got", manifest.CID)) + zap.String("got", manifest.Cid)) return } // Store the dataset size for later use - this indicates success d.mu.Lock() - d.datasetSize = manifest.Manifest.DatasetSize + d.datasetSize = int64(manifest.DatasetSize) d.mu.Unlock() // Success! Close the channel to signal completion diff --git a/communities/codex_index_downloader_integration_test.go b/communities/codex_index_downloader_integration_test.go index 0179a30..1680546 100644 --- a/communities/codex_index_downloader_integration_test.go +++ b/communities/codex_index_downloader_integration_test.go @@ -30,23 +30,12 @@ type CodexIndexDownloaderIntegrationTestSuite struct { suite.Suite client *communities.CodexClient testDir string - host string - port string logger *zap.Logger } // SetupSuite runs once before all tests in the suite func (suite *CodexIndexDownloaderIntegrationTestSuite) SetupSuite() { - suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost") - suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8001") - suite.client = communities.NewCodexClient(suite.host, suite.port) - - // Optional request timeout override - if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { - if d, err := time.ParseDuration(ms + "ms"); err == nil { - suite.client.SetRequestTimeout(d) - } - } + suite.client = communities.NewCodexClientTest(suite.T()) // Create logger suite.logger, _ = zap.NewDevelopment() diff --git a/communities/codex_index_downloader_test.go b/communities/codex_index_downloader_test.go index 5bee170..3a3e00b 100644 --- a/communities/codex_index_downloader_test.go +++ b/communities/codex_index_downloader_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/codex-storage/codex-go-bindings/codex" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -79,12 +80,12 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_SuccessClosesChannel filePath := filepath.Join(suite.testDir, "index.bin") // Setup mock to return a successful manifest - expectedManifest := &communities.CodexManifest{ - CID: testCid, + expectedManifest := codex.Manifest{ + Cid: testCid, } - expectedManifest.Manifest.DatasetSize = 1024 - expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID" - expectedManifest.Manifest.BlockSize = 65536 + expectedManifest.DatasetSize = 1024 + expectedManifest.TreeCid = "zDvZRwzmTreeCID" + expectedManifest.BlockSize = 65536 suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). @@ -119,7 +120,7 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_ErrorDoesNotCloseCha // Setup mock to return an error suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). - Return(nil, errors.New("fetch error")) + Return(codex.Manifest{}, errors.New("fetch error")) // Create downloader downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger) @@ -154,10 +155,10 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_CidMismatchDoesNotCl filePath := filepath.Join(suite.testDir, "index.bin") // Setup mock to return a manifest with different CID - mismatchedManifest := &communities.CodexManifest{ - CID: differentCid, // Different CID! + mismatchedManifest := codex.Manifest{ + Cid: differentCid, // Different CID! } - mismatchedManifest.Manifest.DatasetSize = 1024 + mismatchedManifest.DatasetSize = 1024 suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). @@ -198,12 +199,12 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_Cancellation() { fetchCalled := make(chan struct{}) suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). - DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) { + DoAndReturn(func(ctx context.Context, cid string) (codex.Manifest, error) { close(fetchCalled) // Signal that fetch was called // Wait for context cancellation <-ctx.Done() - return nil, ctx.Err() + return codex.Manifest{}, ctx.Err() }) // Create downloader @@ -250,11 +251,11 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize() expectedSize := int64(2048) // Setup mock to return a manifest with specific dataset size - expectedManifest := &communities.CodexManifest{ - CID: testCid, + expectedManifest := codex.Manifest{ + Cid: testCid, } - expectedManifest.Manifest.DatasetSize = expectedSize - expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID" + expectedManifest.DatasetSize = int(expectedSize) + expectedManifest.TreeCid = "zDvZRwzmTreeCID" suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). @@ -278,7 +279,7 @@ func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize() } // Verify dataset size was recorded correctly - assert.Equal(suite.T(), expectedSize, downloader.GetDatasetSize(), "Dataset size should match manifest") + assert.Equal(suite.T(), int64(expectedSize), downloader.GetDatasetSize(), "Dataset size should match manifest") suite.T().Logf("✅ Dataset size correctly recorded: %d", downloader.GetDatasetSize()) // Verify no error was recorded @@ -500,13 +501,13 @@ func (suite *CodexIndexDownloaderTestSuite) TestDownloadIndexFile_ErrorHandling( func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() { testCid := "zDvZRwzmTestCID123" filePath := filepath.Join(suite.testDir, "index.bin") - expectedSize := int64(4096) + expectedSize := 4096 // Setup mock to return a manifest - expectedManifest := &communities.CodexManifest{ - CID: testCid, + expectedManifest := codex.Manifest{ + Cid: testCid, } - expectedManifest.Manifest.DatasetSize = expectedSize + expectedManifest.DatasetSize = expectedSize suite.mockClient.EXPECT(). FetchManifestWithContext(gomock.Any(), testCid). @@ -523,6 +524,6 @@ func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() { <-manifestChan // Now Length should return the dataset size - assert.Equal(suite.T(), expectedSize, downloader.Length(), "Length should return dataset size") + assert.Equal(suite.T(), int64(expectedSize), downloader.Length(), "Length should return dataset size") suite.T().Logf("✅ Length() correctly returns dataset size: %d", downloader.Length()) } diff --git a/communities/codex_manifest.go b/communities/codex_manifest.go deleted file mode 100644 index 3b509a1..0000000 --- a/communities/codex_manifest.go +++ /dev/null @@ -1,14 +0,0 @@ -package communities - -// CodexManifest represents the manifest structure returned by Codex API -type CodexManifest struct { - CID string `json:"cid"` - Manifest struct { - TreeCid string `json:"treeCid"` - DatasetSize int64 `json:"datasetSize"` - BlockSize int `json:"blockSize"` - Protected bool `json:"protected"` - Filename string `json:"filename"` - Mimetype string `json:"mimetype"` - } `json:"manifest"` -} diff --git a/communities/mock/codex_client_interface.go b/communities/mock/codex_client_interface.go index 792c378..c7db270 100644 --- a/communities/mock/codex_client_interface.go +++ b/communities/mock/codex_client_interface.go @@ -11,11 +11,11 @@ package mock_communities import ( context "context" - communities "go-codex-client/communities" io "io" reflect "reflect" time "time" + "github.com/codex-storage/codex-go-bindings/codex" gomock "go.uber.org/mock/gomock" ) @@ -72,10 +72,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) DownloadWithContext(ctx, cid, ou } // FetchManifestWithContext mocks base method. -func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) { +func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (codex.Manifest, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FetchManifestWithContext", ctx, cid) - ret0, _ := ret[0].(*communities.CodexManifest) + ret0, _ := ret[0].(codex.Manifest) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -156,10 +156,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) SetRequestTimeout(timeout any) * } // TriggerDownload mocks base method. -func (m *MockCodexClientInterface) TriggerDownload(cid string) (*communities.CodexManifest, error) { +func (m *MockCodexClientInterface) TriggerDownload(cid string) (codex.Manifest, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "TriggerDownload", cid) - ret0, _ := ret[0].(*communities.CodexManifest) + ret0, _ := ret[0].(codex.Manifest) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -171,10 +171,10 @@ func (mr *MockCodexClientInterfaceMockRecorder) TriggerDownload(cid any) *gomock } // TriggerDownloadWithContext mocks base method. -func (m *MockCodexClientInterface) TriggerDownloadWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) { +func (m *MockCodexClientInterface) TriggerDownloadWithContext(ctx context.Context, cid string) (codex.Manifest, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "TriggerDownloadWithContext", ctx, cid) - ret0, _ := ret[0].(*communities.CodexManifest) + ret0, _ := ret[0].(codex.Manifest) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/communities/testutil.go b/communities/testutil.go new file mode 100644 index 0000000..fbe8aff --- /dev/null +++ b/communities/testutil.go @@ -0,0 +1,37 @@ +package communities + +import ( + "testing" + + "github.com/codex-storage/codex-go-bindings/codex" +) + +func NewCodexClientTest(t *testing.T) *CodexClient { + client, err := NewCodexClient(codex.Config{ + DataDir: t.TempDir(), + LogFormat: codex.LogFormatNoColors, + MetricsEnabled: false, + BlockRetries: 5, + DiscoveryPort: 8092, + }) + if err != nil { + t.Fatalf("Failed to create Codex node: %v", err) + } + + err = client.Start() + if err != nil { + t.Fatalf("Failed to start Codex node: %v", err) + } + + t.Cleanup(func() { + if err := client.Stop(); err != nil { + t.Logf("cleanup codex: %v", err) + } + + if err := client.Destroy(); err != nil { + t.Logf("cleanup codex: %v", err) + } + }) + + return client +} diff --git a/go.mod b/go.mod index e3e6a5f..f4b2398 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go-codex-client -go 1.23.0 +go 1.24.0 require ( github.com/stretchr/testify v1.11.1 @@ -10,6 +10,7 @@ require ( ) require ( + github.com/codex-storage/codex-go-bindings v0.0.22 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.uber.org/multierr v1.10.0 // indirect diff --git a/go.sum b/go.sum index 61d1196..f414298 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/codex-storage/codex-go-bindings v0.0.22 h1:53nOqLzgfvR3KdghFAKDoREoW+n12ewvNf8Zf3Pdobc= +github.com/codex-storage/codex-go-bindings v0.0.22/go.mod h1:hP/n9iDZqQP4MytkgUepl3yMMsZy5Jbk9lQbbbVJ51Q= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=