From 626ef22d4949a01deda8bc3f5eaeb59e65b349e7 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Fri, 24 Oct 2025 02:22:00 +0200 Subject: [PATCH] FetchManifestWithContext extracted from index downloader and added to CodexClient --- communities/codex_client.go | 49 +++-- communities/codex_client_integration_test.go | 92 ++++++++++ communities/codex_client_interface.go | 3 + communities/codex_client_test.go | 180 +++++++++++++++++++ communities/codex_index_downloader.go | 32 +--- communities/mock/codex_client_interface.go | 15 ++ 6 files changed, 332 insertions(+), 39 deletions(-) diff --git a/communities/codex_client.go b/communities/codex_client.go index 74f33f1..c534d38 100644 --- a/communities/codex_client.go +++ b/communities/codex_client.go @@ -23,6 +23,18 @@ type CodexClient struct { Client *http.Client } +type CodexManifest struct { + CID string `json:"cid"` + Manifest struct { + TreeCid string `json:"treeCid"` + DatasetSize int64 `json:"datasetSize"` + BlockSize int `json:"blockSize"` + Protected bool `json:"protected"` + Filename string `json:"filename"` + Mimetype string `json:"mimetype"` + } `json:"manifest"` +} + // NewCodexClient creates a new Codex client func NewCodexClient(host string, port string) *CodexClient { return &CodexClient{ @@ -179,17 +191,32 @@ func (c *CodexClient) LocalDownloadWithContext(ctx context.Context, cid string, return c.copyWithContext(ctx, output, resp.Body) } -// CodexManifest represents the manifest returned by async download -type CodexManifest struct { - CID string `json:"cid"` - Manifest struct { - TreeCid string `json:"treeCid"` - DatasetSize int64 `json:"datasetSize"` - BlockSize int `json:"blockSize"` - Protected bool `json:"protected"` - Filename string `json:"filename"` - Mimetype string `json:"mimetype"` - } `json:"manifest"` +func (c *CodexClient) FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) { + url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/manifest", c.BaseURL, cid) + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := c.Client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch manifest from codex: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("codex fetch manifest failed with status %d: %s", resp.StatusCode, string(body)) + } + + // Parse JSON response containing manifest + var manifest CodexManifest + if err := json.NewDecoder(resp.Body).Decode(&manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest: %w", err) + } + + return &manifest, nil } func (c *CodexClient) TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) { diff --git a/communities/codex_client_integration_test.go b/communities/codex_client_integration_test.go index cbeae3a..ef42c32 100644 --- a/communities/codex_client_integration_test.go +++ b/communities/codex_client_integration_test.go @@ -213,6 +213,98 @@ func TestIntegration_TriggerDownload(t *testing.T) { } } +func TestIntegration_FetchManifest(t *testing.T) { + host := getenv("CODEX_HOST", "localhost") + port := getenv("CODEX_API_PORT", "8080") + client := communities.NewCodexClient(host, port) + + // Optional request timeout override + if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" { + if d, err := time.ParseDuration(ms + "ms"); err == nil { + client.SetRequestTimeout(d) + } + } + + // Generate random payload to ensure proper round-trip verification + payload := make([]byte, 1024) + if _, err := rand.Read(payload); err != nil { + t.Fatalf("failed to generate random payload: %v", err) + } + t.Logf("Generated payload (first 32 bytes hex): %s", hex.EncodeToString(payload[:32])) + + cid, err := client.Upload(bytes.NewReader(payload), "fetch-manifest-test.bin") + if err != nil { + t.Fatalf("upload failed: %v", err) + } + t.Logf("Upload successful, CID: %s", cid) + + // Clean up after test + defer func() { + if err := client.RemoveCid(cid); err != nil { + t.Logf("Warning: Failed to remove CID %s: %v", cid, err) + } + }() + + // Verify existence via HasCid first + exists, err := client.HasCid(cid) + if err != nil { + t.Fatalf("HasCid failed: %v", err) + } + if !exists { + t.Fatalf("HasCid returned false for uploaded CID %s", cid) + } + t.Logf("HasCid confirmed existence of CID: %s", cid) + + // Fetch manifest with context timeout + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + manifest, err := client.FetchManifestWithContext(ctx, cid) + if err != nil { + t.Fatalf("FetchManifestWithContext failed: %v", err) + } + t.Logf("FetchManifest successful, manifest CID: %s", manifest.CID) + + // Verify manifest properties + if manifest.CID != cid { + t.Errorf("Manifest CID mismatch: expected %s, got %s", cid, manifest.CID) + } + + // Verify manifest has expected fields + if manifest.Manifest.TreeCid == "" { + t.Error("Expected TreeCid to be non-empty") + } + t.Logf("Manifest TreeCid: %s", manifest.Manifest.TreeCid) + + if manifest.Manifest.DatasetSize <= 0 { + t.Errorf("Expected DatasetSize > 0, got %d", manifest.Manifest.DatasetSize) + } + t.Logf("Manifest DatasetSize: %d", manifest.Manifest.DatasetSize) + + if manifest.Manifest.BlockSize <= 0 { + t.Errorf("Expected BlockSize > 0, got %d", manifest.Manifest.BlockSize) + } + t.Logf("Manifest BlockSize: %d", manifest.Manifest.BlockSize) + + if manifest.Manifest.Filename != "fetch-manifest-test.bin" { + t.Errorf("Expected Filename 'fetch-manifest-test.bin', got '%s'", manifest.Manifest.Filename) + } + t.Logf("Manifest Filename: %s", manifest.Manifest.Filename) + + // Log manifest details for verification + t.Logf("Manifest Protected: %v", manifest.Manifest.Protected) + t.Logf("Manifest Mimetype: %s", manifest.Manifest.Mimetype) + + // Test fetching manifest for non-existent CID (should fail gracefully) + nonExistentCID := "zDvZRwzmNonExistentCID123456789" + _, err = client.FetchManifestWithContext(ctx, nonExistentCID) + if err == nil { + t.Error("Expected error when fetching manifest for non-existent CID, got nil") + } else { + t.Logf("Expected error for non-existent CID: %v", err) + } +} + func getenv(k, def string) string { if v := os.Getenv(k); v != "" { return v diff --git a/communities/codex_client_interface.go b/communities/codex_client_interface.go index 971315e..fe34395 100644 --- a/communities/codex_client_interface.go +++ b/communities/codex_client_interface.go @@ -31,6 +31,9 @@ type CodexClientInterface interface { TriggerDownload(cid string) (*CodexManifest, error) TriggerDownloadWithContext(ctx context.Context, cid string) (*CodexManifest, error) + // Manifest methods + FetchManifestWithContext(ctx context.Context, cid string) (*CodexManifest, error) + // CID management methods HasCid(cid string) (bool, error) RemoveCid(cid string) error diff --git a/communities/codex_client_test.go b/communities/codex_client_test.go index 8023606..bb7cfcb 100644 --- a/communities/codex_client_test.go +++ b/communities/codex_client_test.go @@ -583,3 +583,183 @@ func TestLocalDownloadWithContext_Cancellation(t *testing.T) { } } } + +func TestFetchManifestWithContext_Success(t *testing.T) { + testCid := "zDvZRwzmTestCID" + expectedManifest := `{ + "cid": "zDvZRwzmTestCID", + "manifest": { + "treeCid": "zDvZRwzmTreeCID123", + "datasetSize": 1024, + "blockSize": 256, + "protected": true, + "filename": "test-file.bin", + "mimetype": "application/octet-stream" + } + }` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + expectedPath := fmt.Sprintf("/api/codex/v1/data/%s/network/manifest", testCid) + if r.URL.Path != expectedPath { + w.WriteHeader(http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(expectedManifest)) + })) + defer server.Close() + + client := communities.NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + ctx := context.Background() + manifest, err := client.FetchManifestWithContext(ctx, testCid) + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if manifest == nil { + t.Fatal("Expected manifest, got nil") + } + + if manifest.CID != testCid { + t.Errorf("Expected CID %s, got %s", testCid, manifest.CID) + } + + if manifest.Manifest.TreeCid != "zDvZRwzmTreeCID123" { + t.Errorf("Expected TreeCid %s, got %s", "zDvZRwzmTreeCID123", manifest.Manifest.TreeCid) + } + + if manifest.Manifest.DatasetSize != 1024 { + t.Errorf("Expected DatasetSize %d, got %d", 1024, manifest.Manifest.DatasetSize) + } + + if manifest.Manifest.BlockSize != 256 { + t.Errorf("Expected BlockSize %d, got %d", 256, manifest.Manifest.BlockSize) + } + + if !manifest.Manifest.Protected { + t.Error("Expected Protected to be true, got false") + } + + if manifest.Manifest.Filename != "test-file.bin" { + t.Errorf("Expected Filename %s, got %s", "test-file.bin", manifest.Manifest.Filename) + } + + if manifest.Manifest.Mimetype != "application/octet-stream" { + t.Errorf("Expected Mimetype %s, got %s", "application/octet-stream", manifest.Manifest.Mimetype) + } +} + +func TestFetchManifestWithContext_RequestError(t *testing.T) { + client := communities.NewCodexClient("invalid-host", "8080") + + ctx := context.Background() + manifest, err := client.FetchManifestWithContext(ctx, "test-cid") + if err == nil { + t.Fatal("Expected error for invalid host, got nil") + } + if manifest != nil { + t.Fatal("Expected nil manifest on error, got non-nil") + } + + if !strings.Contains(err.Error(), "failed to fetch manifest from codex") { + t.Errorf("Expected 'failed to fetch manifest from codex' in error message, got: %v", err) + } +} + +func TestFetchManifestWithContext_HTTPError(t *testing.T) { + testCid := "zDvZRwzmTestCID" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + w.Write([]byte("Manifest not found")) + })) + defer server.Close() + + client := communities.NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + ctx := context.Background() + manifest, err := client.FetchManifestWithContext(ctx, testCid) + if err == nil { + t.Fatal("Expected error for HTTP 404, got nil") + } + if manifest != nil { + t.Fatal("Expected nil manifest on error, got non-nil") + } + + if !strings.Contains(err.Error(), "404") { + t.Errorf("Expected '404' in error message, got: %v", err) + } +} + +func TestFetchManifestWithContext_JSONParseError(t *testing.T) { + testCid := "zDvZRwzmTestCID" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte("invalid json {")) + })) + defer server.Close() + + client := communities.NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + ctx := context.Background() + manifest, err := client.FetchManifestWithContext(ctx, testCid) + if err == nil { + t.Fatal("Expected error for invalid JSON, got nil") + } + if manifest != nil { + t.Fatal("Expected nil manifest on JSON parse error, got non-nil") + } + + if !strings.Contains(err.Error(), "failed to parse manifest") { + t.Errorf("Expected 'failed to parse manifest' in error message, got: %v", err) + } +} + +func TestFetchManifestWithContext_Cancellation(t *testing.T) { + testCid := "zDvZRwzmTestCID" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Simulate a slow response + time.Sleep(100 * time.Millisecond) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"cid": "test"}`)) + })) + defer server.Close() + + client := communities.NewCodexClient("localhost", "8080") + client.BaseURL = server.URL + + // Create a context with a very short timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + manifest, err := client.FetchManifestWithContext(ctx, testCid) + if err == nil { + t.Fatal("Expected context cancellation error, got nil") + } + if manifest != nil { + t.Fatal("Expected nil manifest on cancellation, got non-nil") + } + + // Accept either canceled or deadline exceeded depending on timing + if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + // net/http may wrap the context error; check error string as a fallback + es := err.Error() + if !(es == context.Canceled.Error() || es == context.DeadlineExceeded.Error()) { + t.Fatalf("expected context cancellation, got: %v", err) + } + } +} diff --git a/communities/codex_index_downloader.go b/communities/codex_index_downloader.go index 45c01eb..e5a9fe4 100644 --- a/communities/codex_index_downloader.go +++ b/communities/codex_index_downloader.go @@ -2,10 +2,7 @@ package communities import ( "context" - "encoding/json" - "fmt" "io" - "net/http" "os" ) @@ -73,41 +70,20 @@ func (d *CodexIndexDownloader) GotManifest() <-chan struct{} { }() // Fetch manifest from Codex - url := fmt.Sprintf("%s/api/codex/v1/data/%s/network/manifest", d.codexClient.BaseURL, d.indexCid) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return - } - - resp, err := d.codexClient.Client.Do(req) + manifest, err := d.codexClient.FetchManifestWithContext(ctx, d.indexCid) if err != nil { // Don't close channel on error - let timeout handle it - return - } - defer resp.Body.Close() - - // Check if request was successful - if resp.StatusCode != http.StatusOK { - // Don't close channel on error - let timeout handle it - return - } - - // Parse the JSON response - var manifestResp ManifestResponse - if err := json.NewDecoder(resp.Body).Decode(&manifestResp); err != nil { - // Don't close channel on error - let timeout handle it + // This is to fit better in the original status-go app return } // Verify that the CID matches our configured indexCid - if manifestResp.CID != d.indexCid { - // Don't close channel on error - let timeout handle it + if manifest.CID != d.indexCid { return } // Store the dataset size for later use - this indicates success - d.datasetSize = manifestResp.Manifest.DatasetSize + d.datasetSize = manifest.Manifest.DatasetSize // Success! Close the channel to signal completion close(ch) diff --git a/communities/mock/codex_client_interface.go b/communities/mock/codex_client_interface.go index 5d58836..792c378 100644 --- a/communities/mock/codex_client_interface.go +++ b/communities/mock/codex_client_interface.go @@ -71,6 +71,21 @@ func (mr *MockCodexClientInterfaceMockRecorder) DownloadWithContext(ctx, cid, ou return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DownloadWithContext", reflect.TypeOf((*MockCodexClientInterface)(nil).DownloadWithContext), ctx, cid, output) } +// FetchManifestWithContext mocks base method. +func (m *MockCodexClientInterface) FetchManifestWithContext(ctx context.Context, cid string) (*communities.CodexManifest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FetchManifestWithContext", ctx, cid) + ret0, _ := ret[0].(*communities.CodexManifest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FetchManifestWithContext indicates an expected call of FetchManifestWithContext. +func (mr *MockCodexClientInterfaceMockRecorder) FetchManifestWithContext(ctx, cid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchManifestWithContext", reflect.TypeOf((*MockCodexClientInterface)(nil).FetchManifestWithContext), ctx, cid) +} + // HasCid mocks base method. func (m *MockCodexClientInterface) HasCid(cid string) (bool, error) { m.ctrl.T.Helper()