Renames LocalDownload to TriggerDownload and brings back the original LocalDownload

This commit is contained in:
Marcin Czenko 2025-10-22 03:44:49 +02:00
parent 4dc98d22e3
commit 774660639b
No known key found for this signature in database
GPG Key ID: A0449219BDBA98AE
4 changed files with 196 additions and 21 deletions

View File

@ -18,7 +18,7 @@ A lightweight Go client library for interacting with Codex decentralized storage
The `CodexClient` type in `communities/codex_client.go` is the central abstraction:
- Wraps Codex HTTP API (`/api/codex/v1/data/*`) with Go-idiomatic methods
- Supports both network downloads (`/network/stream`) and local downloads (direct CID access)
- Context-aware operations for cancellation support (`DownloadWithContext`, `LocalDownloadWithContext`)
- Context-aware operations for cancellation support (`DownloadWithContext`, `TriggerDownloadWithContext`)
- All uploads use `application/octet-stream` with `Content-Disposition` header for filenames
### Data Flow Pattern
@ -106,13 +106,17 @@ flag.Parse()
- Unit tests (stdlib) in `communities/codex_client_test.go` cover:
- Upload success (headers validated) returning CID
- Download success to a `bytes.Buffer`
- HasCid existence checks with JSON parsing
- RemoveCid deletion with 204 status validation
- TriggerDownload async operations with manifest parsing
- LocalDownload direct local storage access
- Cancellation: streaming handler exits on client cancel; fast and warning-free
- Run: `go test -v ./communities`
- Integration test (requires Codex node): `communities/codex_client_integration_test.go`
- Build tag-gated: run with `go test -v -tags=integration ./communities -run Integration`
- Env: `CODEX_HOST` (default `localhost`), `CODEX_API_PORT` (default `8080`), optional `CODEX_TIMEOUT_MS`
- Uses random 1KB payload, logs hex preview, uploads, downloads, and verifies equality
- Includes LocalDownload test: uploads→triggers async download→polls HasCid (10s timeout)→verifies content
- Includes TriggerDownload test: uploads→triggers async download→polls HasCid (10s timeout)→verifies content
- All tests use `RemoveCid` cleanup in defer blocks to prevent storage accumulation
- Debug by observing HTTP responses and Codex node logs; client timeout defaults to 60s

View File

@ -73,8 +73,8 @@ func (c *CodexClient) Download(cid string, output io.Writer) error {
return c.DownloadWithContext(context.Background(), cid, output)
}
func (c *CodexClient) LocalDownload(cid string) (*CodexManifest, error) {
return c.LocalDownloadWithContext(context.Background(), cid)
func (c *CodexClient) TriggerDownload(cid string) (*CodexManifest, error) {
return c.TriggerDownloadWithContext(context.Background(), cid)
}
func (c *CodexClient) HasCid(cid string) (bool, error) {
@ -152,6 +152,33 @@ func (c *CodexClient) DownloadWithContext(ctx context.Context, cid string, outpu
return c.copyWithContext(ctx, output, resp.Body)
}
func (c *CodexClient) LocalDownload(cid string, output io.Writer) error {
return c.LocalDownloadWithContext(context.Background(), cid, output)
}
func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string, output io.Writer) error {
url := fmt.Sprintf("%s/api/codex/v1/data/%s", c.BaseURL, cid)
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
resp, err := c.Client.Do(req)
if err != nil {
return fmt.Errorf("failed to download from codex: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("codex download failed with status %d: %s", resp.StatusCode, string(body))
}
// Use context-aware copy for cancellable streaming
return c.copyWithContext(ctx, output, resp.Body)
}
// CodexManifest represents the manifest returned by async download
type CodexManifest struct {
CID string `json:"cid"`
@ -165,7 +192,7 @@ type CodexManifest struct {
} `json:"manifest"`
}
func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) {
func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) {
url := fmt.Sprintf("%s/api/codex/v1/data/%s/network", c.BaseURL, cid)
req, err := http.NewRequestWithContext(ctx, "POST", url, nil)

View File

@ -123,7 +123,7 @@ func TestIntegration_CheckNonExistingCID(t *testing.T) {
t.Logf("HasCid confirmed CID is no longer present: %s", cid)
}
func TestIntegration_LocalDownload(t *testing.T) {
func TestIntegration_TriggerDownload(t *testing.T) {
host := getenv("CODEX_HOST", "localhost")
port := getenv("CODEX_API_PORT", "8001") // Use port 8001 as specified by user
client := NewCodexClient(host, port)
@ -157,9 +157,9 @@ func TestIntegration_LocalDownload(t *testing.T) {
}()
// Trigger async download
manifest, err := client.LocalDownload(cid)
manifest, err := client.TriggerDownload(cid)
if err != nil {
t.Fatalf("LocalDownload failed: %v", err)
t.Fatalf("TriggerDownload failed: %v", err)
}
t.Logf("Async download triggered, manifest CID: %s", manifest.CID)
@ -192,13 +192,13 @@ func TestIntegration_LocalDownload(t *testing.T) {
t.Fatalf("Timeout waiting for CID to be available locally after 10 seconds")
}
// Now download the actual content and verify it matches
// Now download the actual content from local storage and verify it matches
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var downloadBuf bytes.Buffer
if err := client.DownloadWithContext(ctx, cid, &downloadBuf); err != nil {
t.Fatalf("Download after local download failed: %v", err)
if err := client.LocalDownloadWithContext(ctx, cid, &downloadBuf); err != nil {
t.Fatalf("LocalDownload after trigger download failed: %v", err)
}
downloadedData := downloadBuf.Bytes()

View File

@ -276,7 +276,7 @@ func TestRemoveCid_Error(t *testing.T) {
}
}
func TestLocalDownloadWithContext_Success(t *testing.T) {
func TestTriggerDownload(t *testing.T) {
const testCid = "zDvZRwzmTestCID"
const expectedManifest = `{
"cid": "zDvZRwzmTestCID",
@ -309,7 +309,7 @@ func TestLocalDownloadWithContext_Success(t *testing.T) {
client.BaseURL = server.URL
ctx := context.Background()
manifest, err := client.LocalDownloadWithContext(ctx, testCid)
manifest, err := client.TriggerDownloadWithContext(ctx, testCid)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -328,7 +328,7 @@ func TestLocalDownloadWithContext_Success(t *testing.T) {
}
}
func TestLocalDownloadWithContext_RequestError(t *testing.T) {
func TestTriggerDownloadWithContext_RequestError(t *testing.T) {
// Create a server and immediately close it to trigger connection error
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
server.Close()
@ -337,7 +337,7 @@ func TestLocalDownloadWithContext_RequestError(t *testing.T) {
client.BaseURL = server.URL
ctx := context.Background()
manifest, err := client.LocalDownloadWithContext(ctx, "zDvZRwzmTestCID")
manifest, err := client.TriggerDownloadWithContext(ctx, "zDvZRwzmRigWseNB7WqmudkKAPgZmrDCE9u5cY4KvCqhRo9Ki")
if err == nil {
t.Fatal("expected error, got nil")
}
@ -346,7 +346,7 @@ func TestLocalDownloadWithContext_RequestError(t *testing.T) {
}
}
func TestLocalDownloadWithContext_JSONParseError(t *testing.T) {
func TestTriggerDownloadWithContext_JSONParseError(t *testing.T) {
const testCid = "zDvZRwzmTestCID"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -361,7 +361,7 @@ func TestLocalDownloadWithContext_JSONParseError(t *testing.T) {
client.BaseURL = server.URL
ctx := context.Background()
manifest, err := client.LocalDownloadWithContext(ctx, testCid)
manifest, err := client.TriggerDownloadWithContext(ctx, testCid)
if err == nil {
t.Fatal("expected JSON parse error, got nil")
}
@ -373,7 +373,7 @@ func TestLocalDownloadWithContext_JSONParseError(t *testing.T) {
}
}
func TestLocalDownloadWithContext_HTTPError(t *testing.T) {
func TestTriggerDownloadWithContext_HTTPError(t *testing.T) {
const testCid = "zDvZRwzmTestCID"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -386,7 +386,7 @@ func TestLocalDownloadWithContext_HTTPError(t *testing.T) {
client.BaseURL = server.URL
ctx := context.Background()
manifest, err := client.LocalDownloadWithContext(ctx, testCid)
manifest, err := client.TriggerDownloadWithContext(ctx, testCid)
if err == nil {
t.Fatal("expected error for 404 status, got nil")
}
@ -398,7 +398,7 @@ func TestLocalDownloadWithContext_HTTPError(t *testing.T) {
}
}
func TestLocalDownloadWithContext_Cancellation(t *testing.T) {
func TestTriggerDownloadWithContext_Cancellation(t *testing.T) {
const testCid = "zDvZRwzmTestCID"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -421,7 +421,7 @@ func TestLocalDownloadWithContext_Cancellation(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
manifest, err := client.LocalDownloadWithContext(ctx, testCid)
manifest, err := client.TriggerDownloadWithContext(ctx, testCid)
if err == nil {
t.Fatal("expected cancellation error, got nil")
}
@ -437,3 +437,147 @@ func TestLocalDownloadWithContext_Cancellation(t *testing.T) {
}
}
}
func TestLocalDownload(t *testing.T) {
testData := []byte("test data for local download")
testCid := "zDvZRwzmTestCID"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Verify request method and path
if r.Method != "GET" {
t.Errorf("Expected GET request, got %s", r.Method)
}
expectedPath := "/api/codex/v1/data/" + testCid
if r.URL.Path != expectedPath {
t.Errorf("Expected path %s, got %s", expectedPath, r.URL.Path)
}
w.WriteHeader(http.StatusOK)
w.Write(testData)
}))
defer server.Close()
client := NewCodexClient("localhost", "8080")
client.BaseURL = server.URL
var buf bytes.Buffer
err := client.LocalDownload(testCid, &buf)
if err != nil {
t.Fatalf("LocalDownload failed: %v", err)
}
if !bytes.Equal(buf.Bytes(), testData) {
t.Errorf("Downloaded data mismatch. Expected %q, got %q", string(testData), buf.String())
}
}
func TestLocalDownloadWithContext_Success(t *testing.T) {
testData := []byte("test data for local download with context")
testCid := "zDvZRwzmTestCID"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Verify request method and path
if r.Method != "GET" {
t.Errorf("Expected GET request, got %s", r.Method)
}
expectedPath := "/api/codex/v1/data/" + testCid
if r.URL.Path != expectedPath {
t.Errorf("Expected path %s, got %s", expectedPath, r.URL.Path)
}
w.WriteHeader(http.StatusOK)
w.Write(testData)
}))
defer server.Close()
client := NewCodexClient("localhost", "8080")
client.BaseURL = server.URL
ctx := context.Background()
var buf bytes.Buffer
err := client.LocalDownloadWithContext(ctx, testCid, &buf)
if err != nil {
t.Fatalf("LocalDownloadWithContext failed: %v", err)
}
if !bytes.Equal(buf.Bytes(), testData) {
t.Errorf("Downloaded data mismatch. Expected %q, got %q", string(testData), buf.String())
}
}
func TestLocalDownloadWithContext_RequestError(t *testing.T) {
// Create a server and immediately close it to trigger connection error
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
server.Close()
client := NewCodexClient("localhost", "8080")
client.BaseURL = server.URL
ctx := context.Background()
var buf bytes.Buffer
err := client.LocalDownloadWithContext(ctx, "zDvZRwzmTestCID", &buf)
if err == nil {
t.Fatal("Expected error due to closed server, got nil")
}
if !strings.Contains(err.Error(), "failed to download from codex") {
t.Errorf("Expected 'failed to download from codex' in error, got: %v", err)
}
}
func TestLocalDownloadWithContext_HTTPError(t *testing.T) {
testCid := "zDvZRwzmTestCID"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("CID not found in local storage"))
}))
defer server.Close()
client := NewCodexClient("localhost", "8080")
client.BaseURL = server.URL
ctx := context.Background()
var buf bytes.Buffer
err := client.LocalDownloadWithContext(ctx, testCid, &buf)
if err == nil {
t.Fatal("Expected error for HTTP 404, got nil")
}
if !strings.Contains(err.Error(), "404") {
t.Errorf("Expected '404' in error message, got: %v", err)
}
}
func TestLocalDownloadWithContext_Cancellation(t *testing.T) {
testCid := "zDvZRwzmTestCID"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Simulate a slow response
time.Sleep(100 * time.Millisecond)
w.WriteHeader(http.StatusOK)
w.Write([]byte("slow response"))
}))
defer server.Close()
client := NewCodexClient("localhost", "8080")
client.BaseURL = server.URL
// Create a context with a very short timeout
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel()
var buf bytes.Buffer
err := client.LocalDownloadWithContext(ctx, testCid, &buf)
if err == nil {
t.Fatal("Expected context cancellation error, got nil")
}
// Accept either canceled or deadline exceeded depending on timing
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
// net/http may wrap the context error; check error string as a fallback
es := err.Error()
if !(es == context.Canceled.Error() || es == context.DeadlineExceeded.Error()) {
t.Fatalf("expected context cancellation, got: %v", err)
}
}
}