adds tests for the index downloader and refactors things a bit

communities/codex_archive_downloader_integration_test.go

@@ -32,8 +32,8 @@ type CodexArchiveDownloaderIntegrationSuite struct {
 // SetupSuite runs once before all tests in the suite
 func (suite *CodexArchiveDownloaderIntegrationSuite) SetupSuite() {
 	// Use port 8001 as specified by the user
-	host := getEnvWithDefault("CODEX_HOST", "localhost")
-	port := getEnvWithDefault("CODEX_API_PORT", "8001")
+	host := communities.GetEnvOrDefault("CODEX_HOST", "localhost")
+	port := communities.GetEnvOrDefault("CODEX_API_PORT", "8001")
 	suite.client = communities.NewCodexClient(host, port)
 
 	// Optional request timeout override
@@ -224,11 +224,3 @@ func (suite *CodexArchiveDownloaderIntegrationSuite) TestFullArchiveDownloadWork
 func TestCodexArchiveDownloaderIntegrationSuite(t *testing.T) {
 	suite.Run(t, new(CodexArchiveDownloaderIntegrationSuite))
 }
-
-// Helper function for environment variables with defaults
-func getEnvWithDefault(k, def string) string {
-	if v := os.Getenv(k); v != "" {
-		return v
-	}
-	return def
-}

communities/codex_client_integration_test.go

@@ -34,8 +34,8 @@ type CodexClientIntegrationTestSuite struct {
 
 // SetupSuite runs once before all tests in the suite
 func (suite *CodexClientIntegrationTestSuite) SetupSuite() {
-	suite.host = getenv("CODEX_HOST", "localhost")
-	suite.port = getenv("CODEX_API_PORT", "8080")
+	suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost")
+	suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8080")
 	suite.client = communities.NewCodexClient(suite.host, suite.port)
 
 	// Optional request timeout override
@@ -116,7 +116,7 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_CheckNonExistingCI
 func (suite *CodexClientIntegrationTestSuite) TestIntegration_TriggerDownload() {
 	// Use port 8001 for this test as specified
 	client := communities.NewCodexClient(suite.host, "8001")
-	
+
 	// Optional request timeout override
 	if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" {
 		if d, err := time.ParseDuration(ms + "ms"); err == nil {
@@ -249,10 +249,3 @@ func (suite *CodexClientIntegrationTestSuite) TestIntegration_FetchManifest() {
 	assert.Error(suite.T(), err, "Expected error when fetching manifest for non-existent CID")
 	suite.T().Logf("Expected error for non-existent CID: %v", err)
 }
-
-func getenv(k, def string) string {
-	if v := os.Getenv(k); v != "" {
-		return v
-	}
-	return def
-}

communities/codex_index_downloader.go

@@ -4,38 +4,29 @@ import (
 	"context"
 	"io"
 	"os"
-)
 
-// ManifestResponse represents the response from Codex manifest API
-type ManifestResponse struct {
-	CID      string `json:"cid"`
-	Manifest struct {
-		TreeCID     string `json:"treeCid"`
-		DatasetSize int64  `json:"datasetSize"`
-		BlockSize   int    `json:"blockSize"`
-		Protected   bool   `json:"protected"`
-		Filename    string `json:"filename"`
-		Mimetype    string `json:"mimetype"`
-	} `json:"manifest"`
-}
+	"go.uber.org/zap"
+)
 
 // CodexIndexDownloader handles downloading index files from Codex storage
 type CodexIndexDownloader struct {
-	codexClient    *CodexClient
+	codexClient    CodexClientInterface
 	indexCid       string
 	filePath       string
 	datasetSize    int64           // stores the dataset size from the manifest
 	bytesCompleted int64           // tracks download progress
 	cancelChan     <-chan struct{} // for cancellation support
+	logger         *zap.Logger
 }
 
 // NewCodexIndexDownloader creates a new index downloader
-func NewCodexIndexDownloader(codexClient *CodexClient, indexCid string, filePath string, cancelChan <-chan struct{}) *CodexIndexDownloader {
+func NewCodexIndexDownloader(codexClient CodexClientInterface, indexCid string, filePath string, cancelChan <-chan struct{}, logger *zap.Logger) *CodexIndexDownloader {
 	return &CodexIndexDownloader{
 		codexClient: codexClient,
 		indexCid:    indexCid,
 		filePath:    filePath,
 		cancelChan:  cancelChan,
+		logger:      logger,
 	}
 }
 
@@ -72,6 +63,9 @@ func (d *CodexIndexDownloader) GotManifest() <-chan struct{} {
 		// Fetch manifest from Codex
 		manifest, err := d.codexClient.FetchManifestWithContext(ctx, d.indexCid)
 		if err != nil {
+			d.logger.Debug("failed to fetch manifest",
+				zap.String("indexCid", d.indexCid),
+				zap.Error(err))
 			// Don't close channel on error - let timeout handle it
 			// This is to fit better in the original status-go app
 			return
@@ -79,6 +73,9 @@ func (d *CodexIndexDownloader) GotManifest() <-chan struct{} {
 
 		// Verify that the CID matches our configured indexCid
 		if manifest.CID != d.indexCid {
+			d.logger.Debug("manifest CID mismatch",
+				zap.String("expected", d.indexCid),
+				zap.String("got", manifest.CID))
 			return
 		}
 
@@ -98,7 +95,7 @@ func (d *CodexIndexDownloader) GetDatasetSize() int64 {
 }
 
 // DownloadIndexFile starts downloading the index file from Codex and writes it to the configured file path
-func (d *CodexIndexDownloader) DownloadIndexFile() error {
+func (d *CodexIndexDownloader) DownloadIndexFile() {
 	// Reset progress counter
 	d.bytesCompleted = 0
 
@@ -127,7 +124,9 @@ func (d *CodexIndexDownloader) DownloadIndexFile() error {
 		// Create the output file
 		file, err := os.Create(d.filePath)
 		if err != nil {
-			// TODO: Consider logging the error or exposing it somehow
+			d.logger.Debug("failed to create file",
+				zap.String("filePath", d.filePath),
+				zap.Error(err))
 			return
 		}
 		defer file.Close()
@@ -141,12 +140,13 @@ func (d *CodexIndexDownloader) DownloadIndexFile() error {
 		// Use CodexClient to download and stream to file with context for cancellation
 		err = d.codexClient.DownloadWithContext(ctx, d.indexCid, progressWriter)
 		if err != nil {
-			// TODO: Consider logging the error or exposing it somehow
+			d.logger.Debug("failed to download index file",
+				zap.String("indexCid", d.indexCid),
+				zap.String("filePath", d.filePath),
+				zap.Error(err))
 			return
 		}
 	}()
-
-	return nil
 }
 
 // BytesCompleted returns the number of bytes downloaded so far

communities/codex_index_downloader_integration_test.go

@@ -0,0 +1,180 @@
+//go:build integration && !disable_torrent
+// +build integration,!disable_torrent
+
+package communities_test
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"go.uber.org/zap"
+
+	"go-codex-client/communities"
+)
+
+// CodexIndexDownloaderIntegrationTestSuite demonstrates testify's suite functionality for CodexIndexDownloader integration tests
+// These tests exercise real network calls against a running Codex node.
+// Required env vars (with defaults):
+//   - CODEX_HOST (default: localhost)
+//   - CODEX_API_PORT (default: 8001)
+//   - CODEX_TIMEOUT_MS (optional; default: 60000)
+type CodexIndexDownloaderIntegrationTestSuite struct {
+	suite.Suite
+	client  *communities.CodexClient
+	testDir string
+	host    string
+	port    string
+	logger  *zap.Logger
+}
+
+// SetupSuite runs once before all tests in the suite
+func (suite *CodexIndexDownloaderIntegrationTestSuite) SetupSuite() {
+	suite.host = communities.GetEnvOrDefault("CODEX_HOST", "localhost")
+	suite.port = communities.GetEnvOrDefault("CODEX_API_PORT", "8001")
+	suite.client = communities.NewCodexClient(suite.host, suite.port)
+
+	// Optional request timeout override
+	if ms := os.Getenv("CODEX_TIMEOUT_MS"); ms != "" {
+		if d, err := time.ParseDuration(ms + "ms"); err == nil {
+			suite.client.SetRequestTimeout(d)
+		}
+	}
+
+	// Create logger
+	suite.logger, _ = zap.NewDevelopment()
+}
+
+// SetupTest runs before each test
+func (suite *CodexIndexDownloaderIntegrationTestSuite) SetupTest() {
+	// Create a temporary directory for test files
+	var err error
+	suite.testDir, err = os.MkdirTemp("", "codex-index-integration-*")
+	require.NoError(suite.T(), err)
+}
+
+// TearDownTest runs after each test
+func (suite *CodexIndexDownloaderIntegrationTestSuite) TearDownTest() {
+	// Clean up test directory
+	if suite.testDir != "" {
+		os.RemoveAll(suite.testDir)
+	}
+}
+
+// TestCodexIndexDownloaderIntegrationTestSuite runs the integration test suite
+func TestCodexIndexDownloaderIntegrationTestSuite(t *testing.T) {
+	suite.Run(t, new(CodexIndexDownloaderIntegrationTestSuite))
+}
+
+func (suite *CodexIndexDownloaderIntegrationTestSuite) TestIntegration_GotManifest() {
+	// Generate random payload to create a test file
+	payload := make([]byte, 2048)
+	_, err := rand.Read(payload)
+	require.NoError(suite.T(), err, "failed to generate random payload")
+	suite.T().Logf("Generated payload (first 32 bytes hex): %s", hex.EncodeToString(payload[:32]))
+
+	// Upload the data to Codex
+	cid, err := suite.client.Upload(bytes.NewReader(payload), "index-manifest-test.bin")
+	require.NoError(suite.T(), err, "upload failed")
+	suite.T().Logf("Upload successful, CID: %s", cid)
+
+	// Clean up after test
+	defer func() {
+		if err := suite.client.RemoveCid(cid); err != nil {
+			suite.T().Logf("Warning: Failed to remove CID %s: %v", cid, err)
+		}
+	}()
+
+	// Create downloader with cancel channel
+	cancelChan := make(chan struct{})
+	defer close(cancelChan)
+
+	filePath := filepath.Join(suite.testDir, "test-index.bin")
+	downloader := communities.NewCodexIndexDownloader(suite.client, cid, filePath, cancelChan, suite.logger)
+
+	// Test GotManifest
+	manifestChan := downloader.GotManifest()
+
+	// Wait for manifest to be fetched (with timeout)
+	select {
+	case <-manifestChan:
+		suite.T().Log("✅ Manifest fetched successfully")
+	case <-time.After(10 * time.Second):
+		suite.T().Fatal("Timeout waiting for manifest to be fetched")
+	}
+
+	// Verify dataset size was recorded
+	datasetSize := downloader.GetDatasetSize()
+	assert.Greater(suite.T(), datasetSize, int64(0), "Dataset size should be greater than 0")
+	suite.T().Logf("Dataset size from manifest: %d bytes", datasetSize)
+
+	// Verify Length returns the same value
+	assert.Equal(suite.T(), datasetSize, downloader.Length(), "Length() should return dataset size")
+}
+
+func (suite *CodexIndexDownloaderIntegrationTestSuite) TestIntegration_DownloadIndexFile() {
+	// Generate random payload
+	payload := make([]byte, 1024)
+	_, err := rand.Read(payload)
+	require.NoError(suite.T(), err, "failed to generate random payload")
+	suite.T().Logf("Generated payload (first 32 bytes hex): %s", hex.EncodeToString(payload[:32]))
+
+	// Upload the data to Codex
+	cid, err := suite.client.Upload(bytes.NewReader(payload), "index-download-test.bin")
+	require.NoError(suite.T(), err, "upload failed")
+	suite.T().Logf("Upload successful, CID: %s", cid)
+
+	// Clean up after test
+	defer func() {
+		if err := suite.client.RemoveCid(cid); err != nil {
+			suite.T().Logf("Warning: Failed to remove CID %s: %v", cid, err)
+		}
+	}()
+
+	// Create downloader
+	cancelChan := make(chan struct{})
+	defer close(cancelChan)
+
+	filePath := filepath.Join(suite.testDir, "downloaded-index.bin")
+	downloader := communities.NewCodexIndexDownloader(suite.client, cid, filePath, cancelChan, suite.logger)
+
+	// First, get the manifest to know the expected size
+	manifestChan := downloader.GotManifest()
+	select {
+	case <-manifestChan:
+		suite.T().Log("Manifest fetched")
+	case <-time.After(10 * time.Second):
+		suite.T().Fatal("Timeout waiting for manifest")
+	}
+
+	expectedSize := downloader.GetDatasetSize()
+	suite.T().Logf("Expected file size: %d bytes", expectedSize)
+
+	// Start the download
+	downloader.DownloadIndexFile()
+
+	// Wait for download to complete by monitoring progress
+	require.Eventually(suite.T(), func() bool {
+		return downloader.BytesCompleted() == expectedSize
+	}, 30*time.Second, 100*time.Millisecond, "Download should complete")
+
+	suite.T().Logf("✅ Download completed: %d/%d bytes", downloader.BytesCompleted(), expectedSize)
+
+	// Verify file exists and has correct size
+	stat, err := os.Stat(filePath)
+	require.NoError(suite.T(), err, "Downloaded file should exist")
+	assert.Equal(suite.T(), expectedSize, stat.Size(), "File size should match dataset size")
+
+	// Verify file contents match original payload
+	downloadedData, err := os.ReadFile(filePath)
+	require.NoError(suite.T(), err, "Should be able to read downloaded file")
+	assert.Equal(suite.T(), payload, downloadedData, "Downloaded data should match original payload")
+	suite.T().Log("✅ Downloaded file contents verified")
+}

communities/codex_index_downloader_test.go

@@ -0,0 +1,460 @@
+//go:build !disable_torrent
+// +build !disable_torrent
+
+package communities_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"go.uber.org/mock/gomock"
+	"go.uber.org/zap"
+
+	"go-codex-client/communities"
+	mock_communities "go-codex-client/communities/mock"
+)
+
+// CodexIndexDownloaderTestSuite demonstrates testify's suite functionality for CodexIndexDownloader tests
+type CodexIndexDownloaderTestSuite struct {
+	suite.Suite
+	ctrl       *gomock.Controller
+	mockClient *mock_communities.MockCodexClientInterface
+	testDir    string
+	cancelChan chan struct{}
+	logger     *zap.Logger
+}
+
+// SetupTest runs before each test method
+func (suite *CodexIndexDownloaderTestSuite) SetupTest() {
+	suite.ctrl = gomock.NewController(suite.T())
+	suite.mockClient = mock_communities.NewMockCodexClientInterface(suite.ctrl)
+
+	// Create a temporary directory for test files
+	var err error
+	suite.testDir, err = os.MkdirTemp("", "codex-index-test-*")
+	require.NoError(suite.T(), err)
+
+	// Create a fresh cancel channel for each test
+	suite.cancelChan = make(chan struct{})
+
+	// Create logger
+	suite.logger, _ = zap.NewDevelopment()
+}
+
+// TearDownTest runs after each test method
+func (suite *CodexIndexDownloaderTestSuite) TearDownTest() {
+	suite.ctrl.Finish()
+
+	// Clean up cancel channel - check if it's still open before closing
+	if suite.cancelChan != nil {
+		select {
+		case <-suite.cancelChan:
+			// Already closed, do nothing
+		default:
+			// Still open, close it
+			close(suite.cancelChan)
+		}
+	}
+
+	// Clean up test directory
+	if suite.testDir != "" {
+		os.RemoveAll(suite.testDir)
+	}
+}
+
+// TestCodexIndexDownloaderTestSuite runs the test suite
+func TestCodexIndexDownloaderTestSuite(t *testing.T) {
+	suite.Run(t, new(CodexIndexDownloaderTestSuite))
+}
+
+// ==================== GotManifest Tests ====================
+
+func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_SuccessClosesChannel() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "index.bin")
+
+	// Setup mock to return a successful manifest
+	expectedManifest := &communities.CodexManifest{
+		CID: testCid,
+	}
+	expectedManifest.Manifest.DatasetSize = 1024
+	expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID"
+	expectedManifest.Manifest.BlockSize = 65536
+
+	suite.mockClient.EXPECT().
+		FetchManifestWithContext(gomock.Any(), testCid).
+		Return(expectedManifest, nil)
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Call GotManifest
+	manifestChan := downloader.GotManifest()
+
+	// Wait for channel to close (with timeout)
+	select {
+	case <-manifestChan:
+		// Success - channel closed as expected
+		suite.T().Log("✅ GotManifest channel closed successfully")
+	case <-time.After(1 * time.Second):
+		suite.T().Fatal("Timeout waiting for GotManifest channel to close")
+	}
+
+	// Verify dataset size was recorded
+	assert.Equal(suite.T(), int64(1024), downloader.GetDatasetSize(), "Dataset size should be recorded")
+}
+
+func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_ErrorDoesNotCloseChannel() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "index.bin")
+
+	// Setup mock to return an error
+	suite.mockClient.EXPECT().
+		FetchManifestWithContext(gomock.Any(), testCid).
+		Return(nil, errors.New("fetch error"))
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Call GotManifest
+	manifestChan := downloader.GotManifest()
+
+	// Channel should NOT close on error
+	select {
+	case <-manifestChan:
+		suite.T().Fatal("GotManifest channel should NOT close on error")
+	case <-time.After(200 * time.Millisecond):
+		// Expected - channel did not close
+		suite.T().Log("✅ GotManifest channel did not close on error (as expected)")
+	}
+
+	// Verify dataset size was NOT recorded (should be 0)
+	assert.Equal(suite.T(), int64(0), downloader.GetDatasetSize(), "Dataset size should be 0 on error")
+}
+
+func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_CidMismatchDoesNotCloseChannel() {
+	testCid := "zDvZRwzmTestCID123"
+	differentCid := "zDvZRwzmDifferentCID456"
+	filePath := filepath.Join(suite.testDir, "index.bin")
+
+	// Setup mock to return a manifest with different CID
+	mismatchedManifest := &communities.CodexManifest{
+		CID: differentCid, // Different CID!
+	}
+	mismatchedManifest.Manifest.DatasetSize = 1024
+
+	suite.mockClient.EXPECT().
+		FetchManifestWithContext(gomock.Any(), testCid).
+		Return(mismatchedManifest, nil)
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Call GotManifest
+	manifestChan := downloader.GotManifest()
+
+	// Channel should NOT close on CID mismatch
+	select {
+	case <-manifestChan:
+		suite.T().Fatal("GotManifest channel should NOT close on CID mismatch")
+	case <-time.After(200 * time.Millisecond):
+		// Expected - channel did not close
+		suite.T().Log("✅ GotManifest channel did not close on CID mismatch (as expected)")
+	}
+
+	// Verify dataset size was NOT recorded (should be 0)
+	assert.Equal(suite.T(), int64(0), downloader.GetDatasetSize(), "Dataset size should be 0 on CID mismatch")
+}
+
+func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_Cancellation() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "index.bin")
+
+	// Setup mock with DoAndReturn to simulate slow response and check for cancellation
+	fetchCalled := make(chan struct{})
+	suite.mockClient.EXPECT().
+		FetchManifestWithContext(gomock.Any(), testCid).
+		DoAndReturn(func(ctx context.Context, cid string) (*communities.CodexManifest, error) {
+			close(fetchCalled) // Signal that fetch was called
+
+			// Wait for context cancellation
+			<-ctx.Done()
+			return nil, ctx.Err()
+		})
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Call GotManifest
+	manifestChan := downloader.GotManifest()
+
+	// Wait for FetchManifestWithContext to be called
+	select {
+	case <-fetchCalled:
+		suite.T().Log("FetchManifestWithContext was called")
+	case <-time.After(1 * time.Second):
+		suite.T().Fatal("Timeout waiting for FetchManifestWithContext to be called")
+	}
+
+	// Now trigger cancellation
+	close(suite.cancelChan)
+
+	// Channel should NOT close on cancellation
+	select {
+	case <-manifestChan:
+		suite.T().Fatal("GotManifest channel should NOT close on cancellation")
+	case <-time.After(200 * time.Millisecond):
+		// Expected - channel did not close
+		suite.T().Log("✅ GotManifest was cancelled and channel did not close (as expected)")
+	}
+
+	// Verify dataset size was NOT recorded
+	assert.Equal(suite.T(), int64(0), downloader.GetDatasetSize(), "Dataset size should be 0 on cancellation")
+}
+
+func (suite *CodexIndexDownloaderTestSuite) TestGotManifest_RecordsDatasetSize() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "index.bin")
+	expectedSize := int64(2048)
+
+	// Setup mock to return a manifest with specific dataset size
+	expectedManifest := &communities.CodexManifest{
+		CID: testCid,
+	}
+	expectedManifest.Manifest.DatasetSize = expectedSize
+	expectedManifest.Manifest.TreeCid = "zDvZRwzmTreeCID"
+
+	suite.mockClient.EXPECT().
+		FetchManifestWithContext(gomock.Any(), testCid).
+		Return(expectedManifest, nil)
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Initially, dataset size should be 0
+	assert.Equal(suite.T(), int64(0), downloader.GetDatasetSize(), "Initial dataset size should be 0")
+
+	// Call GotManifest
+	manifestChan := downloader.GotManifest()
+
+	// Wait for channel to close
+	select {
+	case <-manifestChan:
+		suite.T().Log("GotManifest completed successfully")
+	case <-time.After(1 * time.Second):
+		suite.T().Fatal("Timeout waiting for GotManifest to complete")
+	}
+
+	// Verify dataset size was recorded correctly
+	assert.Equal(suite.T(), expectedSize, downloader.GetDatasetSize(), "Dataset size should match manifest")
+	suite.T().Logf("✅ Dataset size correctly recorded: %d", downloader.GetDatasetSize())
+}
+
+// ==================== DownloadIndexFile Tests ====================
+
+func (suite *CodexIndexDownloaderTestSuite) TestDownloadIndexFile_StoresFileCorrectly() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "downloaded-index.bin")
+	testData := []byte("test index file content with some data")
+
+	// Setup mock to write test data to the provided writer
+	suite.mockClient.EXPECT().
+		DownloadWithContext(gomock.Any(), testCid, gomock.Any()).
+		DoAndReturn(func(ctx context.Context, cid string, w io.Writer) error {
+			_, err := w.Write(testData)
+			return err
+		})
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Start download
+	downloader.DownloadIndexFile()
+
+	// Wait for download to complete (check file existence and size)
+	require.Eventually(suite.T(), func() bool {
+		stat, err := os.Stat(filePath)
+		if err != nil {
+			return false
+		}
+		return stat.Size() == int64(len(testData))
+	}, 2*time.Second, 50*time.Millisecond, "File should be created with correct size")
+
+	// Verify file contents
+	actualData, err := os.ReadFile(filePath)
+	require.NoError(suite.T(), err, "Should be able to read downloaded file")
+	assert.Equal(suite.T(), testData, actualData, "File contents should match")
+	suite.T().Logf("✅ File downloaded successfully to: %s", filePath)
+}
+
+func (suite *CodexIndexDownloaderTestSuite) TestDownloadIndexFile_TracksProgress() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "progress-test.bin")
+	testData := []byte("0123456789") // 10 bytes
+
+	// Setup mock to write test data in chunks
+	suite.mockClient.EXPECT().
+		DownloadWithContext(gomock.Any(), testCid, gomock.Any()).
+		DoAndReturn(func(ctx context.Context, cid string, w io.Writer) error {
+			// Write in 2-byte chunks to simulate streaming
+			for i := 0; i < len(testData); i += 2 {
+				end := i + 2
+				if end > len(testData) {
+					end = len(testData)
+				}
+				_, err := w.Write(testData[i:end])
+				if err != nil {
+					return err
+				}
+				time.Sleep(10 * time.Millisecond) // Small delay to allow progress tracking
+			}
+			return nil
+		})
+
+	// Create downloader and set dataset size
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Start download
+	downloader.DownloadIndexFile()
+
+	// Initially bytes completed should be 0
+	assert.Equal(suite.T(), int64(0), downloader.BytesCompleted(), "Initial bytes completed should be 0")
+
+	// Wait for some progress
+	require.Eventually(suite.T(), func() bool {
+		return downloader.BytesCompleted() > 0
+	}, 1*time.Second, 20*time.Millisecond, "Progress should increase")
+
+	suite.T().Logf("Progress observed: %d bytes", downloader.BytesCompleted())
+
+	// Wait for download to complete
+	require.Eventually(suite.T(), func() bool {
+		return downloader.BytesCompleted() == int64(len(testData))
+	}, 2*time.Second, 50*time.Millisecond, "Should download all bytes")
+
+	assert.Equal(suite.T(), int64(len(testData)), downloader.BytesCompleted(), "All bytes should be downloaded")
+	suite.T().Logf("✅ Download progress tracked correctly: %d/%d bytes", downloader.BytesCompleted(), len(testData))
+}
+
+func (suite *CodexIndexDownloaderTestSuite) TestDownloadIndexFile_Cancellation() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "cancel-test.bin")
+
+	// Setup mock with DoAndReturn to simulate slow download and check for cancellation
+	downloadStarted := make(chan struct{})
+	suite.mockClient.EXPECT().
+		DownloadWithContext(gomock.Any(), testCid, gomock.Any()).
+		DoAndReturn(func(ctx context.Context, cid string, w io.Writer) error {
+			close(downloadStarted) // Signal that download started
+
+			// Simulate slow download with cancellation check
+			for i := 0; i < 100; i++ {
+				select {
+				case <-ctx.Done():
+					return ctx.Err() // Return cancellation error
+				default:
+					_, err := w.Write([]byte("x"))
+					if err != nil {
+						return err
+					}
+					time.Sleep(10 * time.Millisecond)
+				}
+			}
+			return nil
+		})
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Start download
+	downloader.DownloadIndexFile()
+
+	// Wait for download to start
+	select {
+	case <-downloadStarted:
+		suite.T().Log("Download started")
+	case <-time.After(1 * time.Second):
+		suite.T().Fatal("Timeout waiting for download to start")
+	}
+
+	// Trigger cancellation
+	close(suite.cancelChan)
+	suite.T().Log("Cancellation triggered")
+
+	// Wait a bit for cancellation to take effect
+	time.Sleep(200 * time.Millisecond)
+
+	// Verify that download was stopped (bytes completed should be small)
+	bytesCompleted := downloader.BytesCompleted()
+	suite.T().Logf("✅ Download cancelled after %d bytes (should be < 100)", bytesCompleted)
+	assert.Less(suite.T(), bytesCompleted, int64(100), "Download should be cancelled before completing all 100 bytes")
+}
+
+func (suite *CodexIndexDownloaderTestSuite) TestDownloadIndexFile_ErrorHandling() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "error-test.bin")
+
+	// Setup mock to return an error during download
+	suite.mockClient.EXPECT().
+		DownloadWithContext(gomock.Any(), testCid, gomock.Any()).
+		Return(errors.New("download failed"))
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Start download
+	downloader.DownloadIndexFile()
+
+	// Wait a bit for the goroutine to run
+	time.Sleep(200 * time.Millisecond)
+
+	// Verify that no bytes were recorded on error
+	assert.Equal(suite.T(), int64(0), downloader.BytesCompleted(), "No bytes should be recorded on error")
+	suite.T().Log("✅ Error handling: no bytes recorded on download failure")
+
+	// File should exist but be empty (or not exist if creation failed)
+	// This is current behavior - we might want to improve it to clean up on error
+	if stat, err := os.Stat(filePath); err == nil {
+		suite.T().Logf("File exists with size: %d bytes (current behavior)", stat.Size())
+	} else {
+		suite.T().Log("File does not exist (current behavior)")
+	}
+}
+
+func (suite *CodexIndexDownloaderTestSuite) TestLength_ReturnsDatasetSize() {
+	testCid := "zDvZRwzmTestCID123"
+	filePath := filepath.Join(suite.testDir, "index.bin")
+	expectedSize := int64(4096)
+
+	// Setup mock to return a manifest
+	expectedManifest := &communities.CodexManifest{
+		CID: testCid,
+	}
+	expectedManifest.Manifest.DatasetSize = expectedSize
+
+	suite.mockClient.EXPECT().
+		FetchManifestWithContext(gomock.Any(), testCid).
+		Return(expectedManifest, nil)
+
+	// Create downloader
+	downloader := communities.NewCodexIndexDownloader(suite.mockClient, testCid, filePath, suite.cancelChan, suite.logger)
+
+	// Initially, Length should return 0
+	assert.Equal(suite.T(), int64(0), downloader.Length(), "Initial length should be 0")
+
+	// Fetch manifest
+	manifestChan := downloader.GotManifest()
+	<-manifestChan
+
+	// Now Length should return the dataset size
+	assert.Equal(suite.T(), expectedSize, downloader.Length(), "Length should return dataset size")
+	suite.T().Logf("✅ Length() correctly returns dataset size: %d", downloader.Length())
+}

communities/integration_test_helpers.go

@@ -0,0 +1,17 @@
+//go:build integration
+// +build integration
+
+package communities
+
+import "os"
+
+// GetEnvOrDefault returns the value of the environment variable k,
+// or def if the variable is not set or is empty.
+// This helper is shared across all integration test files.
+// It's exported so it can be used by tests in the communities_test package.
+func GetEnvOrDefault(k, def string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return def
+}