From 774660639b796d17cfdf3ec3c0775f2aa2cbb1f4 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 22 Oct 2025 03:44:49 +0200 Subject: [PATCH] Renames LocalDownload to TriggerDownload and brings back the original LocalDownload --- .github/copilot-instructions.md | 8 +- communities/codex_client.go | 33 +++- communities/codex_client_integration_test.go | 12 +- communities/codex_client_test.go | 164 +++++++++++++++++-- 4 files changed, 196 insertions(+), 21 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b3948e0..a65f5bf 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -18,7 +18,7 @@ A lightweight Go client library for interacting with Codex decentralized storage The `CodexClient` type in `communities/codex_client.go` is the central abstraction: - Wraps Codex HTTP API (`/api/codex/v1/data/*`) with Go-idiomatic methods - Supports both network downloads (`/network/stream`) and local downloads (direct CID access) -- Context-aware operations for cancellation support (`DownloadWithContext`, `LocalDownloadWithContext`) +- Context-aware operations for cancellation support (`DownloadWithContext`, `TriggerDownloadWithContext`) - All uploads use `application/octet-stream` with `Content-Disposition` header for filenames ### Data Flow Pattern @@ -106,13 +106,17 @@ flag.Parse() - Unit tests (stdlib) in `communities/codex_client_test.go` cover: - Upload success (headers validated) returning CID - Download success to a `bytes.Buffer` + - HasCid existence checks with JSON parsing + - RemoveCid deletion with 204 status validation + - TriggerDownload async operations with manifest parsing + - LocalDownload direct local storage access - Cancellation: streaming handler exits on client cancel; fast and warning-free - Run: `go test -v ./communities` - Integration test (requires Codex node): `communities/codex_client_integration_test.go` - Build tag-gated: run with `go test -v -tags=integration ./communities -run Integration` - Env: `CODEX_HOST` (default `localhost`), `CODEX_API_PORT` (default `8080`), optional `CODEX_TIMEOUT_MS` - Uses random 1KB payload, logs hex preview, uploads, downloads, and verifies equality - - Includes LocalDownload test: uploads→triggers async download→polls HasCid (10s timeout)→verifies content + - Includes TriggerDownload test: uploads→triggers async download→polls HasCid (10s timeout)→verifies content - All tests use `RemoveCid` cleanup in defer blocks to prevent storage accumulation - Debug by observing HTTP responses and Codex node logs; client timeout defaults to 60s diff --git a/communities/codex_client.go b/communities/codex_client.go index 769349e..74f33f1 100644 --- a/communities/codex_client.go +++ b/communities/codex_client.go @@ -73,8 +73,8 @@ func (c *CodexClient) Download(cid string, output io.Writer) error { return c.DownloadWithContext(context.Background(), cid, output) } -func (c *CodexClient) LocalDownload(cid string) (*CodexManifest, error) { - return c.LocalDownloadWithContext(context.Background(), cid) +func (c *CodexClient) TriggerDownload(cid string) (*CodexManifest, error) { + return c.TriggerDownloadWithContext(context.Background(), cid) } func (c *CodexClient) HasCid(cid string) (bool, error) { @@ -152,6 +152,33 @@ func (c *CodexClient) DownloadWithContext(ctx context.Context, cid string, outpu return c.copyWithContext(ctx, output, resp.Body) } +func (c *CodexClient) LocalDownload(cid string, output io.Writer) error { + return c.LocalDownloadWithContext(context.Background(), cid, output) +} + +func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error { + url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + resp, err := c.Client.Do(req) + if err != nil { + return fmt.Errorf("failed to download from codex: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body)) + } + + // Use context-aware copy for cancellable streaming + return c.copyWithContext(ctx, output, resp.Body) +} + // CodexManifest represents the manifest returned by async download type CodexManifest struct { CID string `json:"cid"` @@ -165,7 +192,7 @@ type CodexManifest struct { } `json:"manifest"` } -func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) { +func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) { url := fmt.Sprintf("%s/api/codex/v1/data/%s/network", c.BaseURL, cid) req, err := http.NewRequestWithContext(ctx, "POST", url, nil) diff --git a/communities/codex_client_integration_test.go b/communities/codex_client_integration_test.go index ac30602..6173103 100644 --- a/communities/codex_client_integration_test.go +++ b/communities/codex_client_integration_test.go @@ -123,7 +123,7 @@ func TestIntegration_CheckNonExistingCID(t *testing.T) { t.Logf("HasCid confirmed CID is no longer present: %s", cid) } -func TestIntegration_LocalDownload(t *testing.T) { +func TestIntegration_TriggerDownload(t *testing.T) { host := getenv("CODEX_HOST", "localhost") port := getenv("CODEX_API_PORT", "8001") // Use port 8001 as specified by user client := NewCodexClient(host, port) @@ -157,9 +157,9 @@ func TestIntegration_LocalDownload(t *testing.T) { }() // Trigger async download - manifest, err := client.LocalDownload(cid) + manifest, err := client.TriggerDownload(cid) if err != nil { - t.Fatalf("LocalDownload failed: %v", err) + t.Fatalf("TriggerDownload failed: %v", err) } t.Logf("Async download triggered, manifest CID: %s", manifest.CID) @@ -192,13 +192,13 @@ func TestIntegration_LocalDownload(t *testing.T) { t.Fatalf("Timeout waiting for CID to be available locally after 10 seconds") } - // Now download the actual content and verify it matches + // Now download the actual content from local storage and verify it matches ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() var downloadBuf bytes.Buffer - if err := client.DownloadWithContext(ctx, cid, &downloadBuf); err != nil { - t.Fatalf("Download after local download failed: %v", err) + if err := client.LocalDownloadWithContext(ctx, cid, &downloadBuf); err != nil { + t.Fatalf("LocalDownload after trigger download failed: %v", err) } downloadedData := downloadBuf.Bytes() diff --git a/communities/codex_client_test.go b/communities/codex_client_test.go index 05a2b8f..970d8a0 100644 --- a/communities/codex_client_test.go +++ b/communities/codex_client_test.go @@ -276,7 +276,7 @@ func TestRemoveCid_Error(t *testing.T) { } } -func TestLocalDownloadWithContext_Success(t *testing.T) { +func TestTriggerDownload(t *testing.T) { const testCid = "zDvZRwzmTestCID" const expectedManifest = `{ "cid": "zDvZRwzmTestCID", @@ -309,7 +309,7 @@ func TestLocalDownloadWithContext_Success(t *testing.T) { client.BaseURL = server.URL ctx := context.Background() - manifest, err := client.LocalDownloadWithContext(ctx, testCid) + manifest, err := client.TriggerDownloadWithContext(ctx, testCid) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -328,7 +328,7 @@ func TestLocalDownloadWithContext_Success(t *testing.T) { } } -func TestLocalDownloadWithContext_RequestError(t *testing.T) { +func TestTriggerDownloadWithContext_RequestError(t *testing.T) { // Create a server and immediately close it to trigger connection error server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) server.Close() @@ -337,7 +337,7 @@ func TestLocalDownloadWithContext_RequestError(t *testing.T) { client.BaseURL = server.URL ctx := context.Background() - manifest, err := client.LocalDownloadWithContext(ctx, "zDvZRwzmTestCID") + manifest, err := client.TriggerDownloadWithContext(ctx, "zDvZRwzmRigWseNB7WqmudkKAPgZmrDCE9u5cY4KvCqhRo9Ki") if err == nil { t.Fatal("expected error, got nil") } @@ -346,7 +346,7 @@ func TestLocalDownloadWithContext_RequestError(t *testing.T) { } } -func TestLocalDownloadWithContext_JSONParseError(t *testing.T) { +func TestTriggerDownloadWithContext_JSONParseError(t *testing.T) { const testCid = "zDvZRwzmTestCID" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -361,7 +361,7 @@ func TestLocalDownloadWithContext_JSONParseError(t *testing.T) { client.BaseURL = server.URL ctx := context.Background() - manifest, err := client.LocalDownloadWithContext(ctx, testCid) + manifest, err := client.TriggerDownloadWithContext(ctx, testCid) if err == nil { t.Fatal("expected JSON parse error, got nil") } @@ -373,7 +373,7 @@ func TestLocalDownloadWithContext_JSONParseError(t *testing.T) { } } -func TestLocalDownloadWithContext_HTTPError(t *testing.T) { +func TestTriggerDownloadWithContext_HTTPError(t *testing.T) { const testCid = "zDvZRwzmTestCID" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -386,7 +386,7 @@ func TestLocalDownloadWithContext_HTTPError(t *testing.T) { client.BaseURL = server.URL ctx := context.Background() - manifest, err := client.LocalDownloadWithContext(ctx, testCid) + manifest, err := client.TriggerDownloadWithContext(ctx, testCid) if err == nil { t.Fatal("expected error for 404 status, got nil") } @@ -398,7 +398,7 @@ func TestLocalDownloadWithContext_HTTPError(t *testing.T) { } } -func TestLocalDownloadWithContext_Cancellation(t *testing.T) { +func TestTriggerDownloadWithContext_Cancellation(t *testing.T) { const testCid = "zDvZRwzmTestCID" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -421,7 +421,7 @@ func TestLocalDownloadWithContext_Cancellation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() - manifest, err := client.LocalDownloadWithContext(ctx, testCid) + manifest, err := client.TriggerDownloadWithContext(ctx, testCid) if err == nil { t.Fatal("expected cancellation error, got nil") } @@ -437,3 +437,147 @@ func TestLocalDownloadWithContext_Cancellation(t *testing.T) { } } } + +func TestLocalDownload(t *testing.T) { + testData := []byte("test data for local download") + testCid := "zDvZRwzmTestCID" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify request method and path + if r.Method != "GET" { + t.Errorf("Expected GET request, got %s", r.Method) + } + expectedPath := "/api/codex/v1/data/" + testCid + if r.URL.Path != expectedPath { + t.Errorf("Expected path %s, got %s", expectedPath, r.URL.Path) + } + + w.WriteHeader(http.StatusOK) + w.Write(testData) + })) + defer server.Close() + + client := NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + var buf bytes.Buffer + err := client.LocalDownload(testCid, &buf) + if err != nil { + t.Fatalf("LocalDownload failed: %v", err) + } + + if !bytes.Equal(buf.Bytes(), testData) { + t.Errorf("Downloaded data mismatch. Expected %q, got %q", string(testData), buf.String()) + } +} + +func TestLocalDownloadWithContext_Success(t *testing.T) { + testData := []byte("test data for local download with context") + testCid := "zDvZRwzmTestCID" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify request method and path + if r.Method != "GET" { + t.Errorf("Expected GET request, got %s", r.Method) + } + expectedPath := "/api/codex/v1/data/" + testCid + if r.URL.Path != expectedPath { + t.Errorf("Expected path %s, got %s", expectedPath, r.URL.Path) + } + + w.WriteHeader(http.StatusOK) + w.Write(testData) + })) + defer server.Close() + + client := NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + ctx := context.Background() + var buf bytes.Buffer + err := client.LocalDownloadWithContext(ctx, testCid, &buf) + if err != nil { + t.Fatalf("LocalDownloadWithContext failed: %v", err) + } + + if !bytes.Equal(buf.Bytes(), testData) { + t.Errorf("Downloaded data mismatch. Expected %q, got %q", string(testData), buf.String()) + } +} + +func TestLocalDownloadWithContext_RequestError(t *testing.T) { + // Create a server and immediately close it to trigger connection error + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + server.Close() + + client := NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + ctx := context.Background() + var buf bytes.Buffer + err := client.LocalDownloadWithContext(ctx, "zDvZRwzmTestCID", &buf) + if err == nil { + t.Fatal("Expected error due to closed server, got nil") + } + + if !strings.Contains(err.Error(), "failed to download from codex") { + t.Errorf("Expected 'failed to download from codex' in error, got: %v", err) + } +} + +func TestLocalDownloadWithContext_HTTPError(t *testing.T) { + testCid := "zDvZRwzmTestCID" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("CID not found in local storage")) + })) + defer server.Close() + + client := NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + ctx := context.Background() + var buf bytes.Buffer + err := client.LocalDownloadWithContext(ctx, testCid, &buf) + if err == nil { + t.Fatal("Expected error for HTTP 404, got nil") + } + + if !strings.Contains(err.Error(), "404") { + t.Errorf("Expected '404' in error message, got: %v", err) + } +} + +func TestLocalDownloadWithContext_Cancellation(t *testing.T) { + testCid := "zDvZRwzmTestCID" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate a slow response + time.Sleep(100 * time.Millisecond) + w.WriteHeader(http.StatusOK) + w.Write([]byte("slow response")) + })) + defer server.Close() + + client := NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + // Create a context with a very short timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + var buf bytes.Buffer + err := client.LocalDownloadWithContext(ctx, testCid, &buf) + if err == nil { + t.Fatal("Expected context cancellation error, got nil") + } + // Accept either canceled or deadline exceeded depending on timing + if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + // net/http may wrap the context error; check error string as a fallback + es := err.Error() + if !(es == context.Canceled.Error() || es == context.DeadlineExceeded.Error()) { + t.Fatalf("expected context cancellation, got: %v", err) + } + } +}