Export fields from download and upload structure

This commit is contained in:
Arnaud 2025-10-14 11:45:23 +02:00
parent 0ea52a8825
commit 773324d620
No known key found for this signature in database
GPG Key ID: 20E40A5D3110766F
8 changed files with 92 additions and 81 deletions

View File

@ -83,7 +83,13 @@ jobs:
git config --global core.symlinks false git config --global core.symlinks false
make update make update
make libcodex make libcodex
- name: Make sure that tests pass
env:
CGO_ENABLED: 1
run: make
run: make test
- name: Package artifacts Linux - name: Package artifacts Linux
if: matrix.target.os == 'ubuntu-latest' || matrix.target.os == 'ubuntu-24.04-arm' if: matrix.target.os == 'ubuntu-latest' || matrix.target.os == 'ubuntu-24.04-arm'
run: | run: |
@ -129,6 +135,5 @@ jobs:
- name: Create release - name: Create release
uses: softprops/action-gh-release@v2 uses: softprops/action-gh-release@v2
with: with:
body_path: ./RELEASE.md
files: dist/** files: dist/**
draft: true draft: true

View File

@ -1,3 +1,9 @@
## v0.0.14 (2025-10-14)
### Notes
- Export fields in upload and download struct
- Fix typo
## v0.0.13 (2025-10-14) ## v0.0.13 (2025-10-14)
### Notes ### Notes

View File

@ -190,6 +190,20 @@ type CodexNode struct {
ctx unsafe.Pointer ctx unsafe.Pointer
} }
type ChunkSize int
func (c ChunkSize) valOrDefault() int {
if c == 0 {
return defaultBlockSize
}
return int(c)
}
func (c ChunkSize) toSizeT() C.size_t {
return C.size_t(c.valOrDefault())
}
// New creates a new Codex node with the provided configuration. // New creates a new Codex node with the provided configuration.
// The node is not started automatically; you need to call CodexStart // The node is not started automatically; you need to call CodexStart
// to start it. // to start it.

View File

@ -31,58 +31,58 @@ import (
"unsafe" "unsafe"
) )
type onDownloadProgressFunc func(read, total int, percent float64, err error) type OnDownloadProgressFunc func(read, total int, percent float64, err error)
// DownloadStreamOptions is used to download a file // DownloadStreamOptions is used to download a file
// in a streaming manner in Codex. // in a streaming manner in Codex.
type DownloadStreamOptions = struct { type DownloadStreamOptions = struct {
// filepath is the path destination used by DownloadStream. // Filepath is the path destination used by DownloadStream.
// If it is set, the content will be written into the specified // If it is set, the content will be written into the specified
// path. // path.
filepath string Filepath string
// chunkSize is the size of each downloaded chunk. Default is to 64 KB. // ChunkSize is the size of each downloaded chunk. Default is to 64 KB.
chunkSize chunckSize ChunkSize ChunkSize
// onProgress is a callback function that is called after each chunk is download with: // OnProgress is a callback function that is called after each chunk is download with:
// - read: the number of bytes downloaded for the last chunk. // - read: the number of bytes downloaded for the last chunk.
// - total: the total number of bytes downloaded so far. // - total: the total number of bytes downloaded so far.
// - percent: the percentage of the total file size that has been downloaded. It is // - percent: the percentage of the total file size that has been downloaded. It is
// determined from `datasetSize`. // determined from `datasetSize`.
// - err: an error, if one occurred. // - err: an error, if one occurred.
onProgress onDownloadProgressFunc OnProgress OnDownloadProgressFunc
// writer is the path destination used by DownloadStream. // Writer is the path destination used by DownloadStream.
// If it is set, the content will be written into the specified // If it is set, the content will be written into the specified
// writer. // Writer.
writer io.Writer Writer io.Writer
// local defines the way to download the content. // Local defines the way to download the content.
// If true, the content will be downloaded from the // If true, the content will be downloaded from the
// local node. // Local node.
// If false (default), the content will be downloaded // If false (default), the content will be downloaded
// from the network. // from the network.
local bool Local bool
// datasetSize is the total size of the dataset being downloaded. // DatasetSize is the total size of the dataset being downloaded.
datasetSize int DatasetSize int
// datasetSizeAuto if true, will fetch the manifest before starting // DatasetSizeAuto if true, will fetch the manifest before starting
// the downloaded to retrive the size of the data. // the downloaded to retrive the size of the data.
datasetSizeAuto bool DatasetSizeAuto bool
} }
// DownloadInitOptions is used to create a download session. // DownloadInitOptions is used to create a download session.
type DownloadInitOptions = struct { type DownloadInitOptions = struct {
// local defines the way to download the content. // Local defines the way to download the content.
// If true, the content will be downloaded from the // If true, the content will be downloaded from the
// local node. // local node.
// If false (default), the content will be downloaded // If false (default), the content will be downloaded
// from the network. // from the network.
local bool Local bool
// chunkSize is the size of each downloaded chunk. Default is to 64 KB. // ChunkSize is the size of each downloaded chunk. Default is to 64 KB.
chunkSize chunckSize ChunkSize ChunkSize
} }
// Manifest is the object containing the information of // Manifest is the object containing the information of
@ -149,14 +149,14 @@ func (node CodexNode) DownloadStream(cid string, options DownloadStreamOptions)
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if options.datasetSizeAuto { if options.DatasetSizeAuto {
manifest, err := node.DownloadManifest(cid) manifest, err := node.DownloadManifest(cid)
if err != nil { if err != nil {
return err return err
} }
options.datasetSize = manifest.DatasetSize options.DatasetSize = manifest.DatasetSize
} }
total := 0 total := 0
@ -165,36 +165,36 @@ func (node CodexNode) DownloadStream(cid string, options DownloadStreamOptions)
return return
} }
if options.writer != nil { if options.Writer != nil {
w := options.writer w := options.Writer
if _, err := w.Write(chunk); err != nil { if _, err := w.Write(chunk); err != nil {
if options.onProgress != nil { if options.OnProgress != nil {
options.onProgress(0, 0, 0.0, err) options.OnProgress(0, 0, 0.0, err)
} }
} }
} }
total += read total += read
if options.onProgress != nil { if options.OnProgress != nil {
var percent = 0.0 var percent = 0.0
if options.datasetSize > 0 { if options.DatasetSize > 0 {
percent = float64(total) / float64(options.datasetSize) * 100.0 percent = float64(total) / float64(options.DatasetSize) * 100.0
} }
options.onProgress(read, total, percent, nil) options.OnProgress(read, total, percent, nil)
} }
} }
var cCid = C.CString(cid) var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid)) defer C.free(unsafe.Pointer(cCid))
var cFilepath = C.CString(options.filepath) var cFilepath = C.CString(options.Filepath)
defer C.free(unsafe.Pointer(cFilepath)) defer C.free(unsafe.Pointer(cFilepath))
var cLocal = C.bool(options.local) var cLocal = C.bool(options.Local)
if C.cGoCodexDownloadStream(node.ctx, cCid, options.chunkSize.toSizeT(), cLocal, cFilepath, bridge.resp) != C.RET_OK { if C.cGoCodexDownloadStream(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, cFilepath, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexDownloadLocal") return bridge.callError("cGoCodexDownloadLocal")
} }
@ -212,9 +212,9 @@ func (node CodexNode) DownloadInit(cid string, options DownloadInitOptions) erro
var cCid = C.CString(cid) var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid)) defer C.free(unsafe.Pointer(cCid))
var cLocal = C.bool(options.local) var cLocal = C.bool(options.Local)
if C.cGoCodexDownloadInit(node.ctx, cCid, options.chunkSize.toSizeT(), cLocal, bridge.resp) != C.RET_OK { if C.cGoCodexDownloadInit(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexDownloadInit") return bridge.callError("cGoCodexDownloadInit")
} }

View File

@ -19,10 +19,10 @@ func TestDownloadStream(t *testing.T) {
totalBytes := 0 totalBytes := 0
finalPercent := 0.0 finalPercent := 0.0
opt := DownloadStreamOptions{ opt := DownloadStreamOptions{
writer: f, Writer: f,
datasetSize: len, DatasetSize: len,
filepath: "testdata/hello.downloaded.writer.txt", Filepath: "testdata/hello.downloaded.writer.txt",
onProgress: func(read, total int, percent float64, err error) { OnProgress: func(read, total int, percent float64, err error) {
if err != nil { if err != nil {
t.Fatalf("Error happening during download: %v\n", err) t.Fatalf("Error happening during download: %v\n", err)
} }
@ -61,8 +61,8 @@ func TestDownloadStreamWithAutosize(t *testing.T) {
totalBytes := 0 totalBytes := 0
finalPercent := 0.0 finalPercent := 0.0
opt := DownloadStreamOptions{ opt := DownloadStreamOptions{
datasetSizeAuto: true, DatasetSizeAuto: true,
onProgress: func(read, total int, percent float64, err error) { OnProgress: func(read, total int, percent float64, err error) {
if err != nil { if err != nil {
t.Fatalf("Error happening during download: %v\n", err) t.Fatalf("Error happening during download: %v\n", err)
} }

View File

@ -65,7 +65,7 @@ func uploadHelper(t *testing.T, codex *CodexNode) (string, int) {
buf := bytes.NewBuffer([]byte("Hello World!")) buf := bytes.NewBuffer([]byte("Hello World!"))
len := buf.Len() len := buf.Len()
cid, err := codex.UploadReader(UploadOptions{filepath: "hello.txt"}, buf) cid, err := codex.UploadReader(UploadOptions{Filepath: "hello.txt"}, buf)
if err != nil { if err != nil {
t.Fatalf("Error happened during upload: %v\n", err) t.Fatalf("Error happened during upload: %v\n", err)
} }

View File

@ -35,33 +35,19 @@ import (
const defaultBlockSize = 1024 * 64 const defaultBlockSize = 1024 * 64
type onUploadProgressFunc func(read, total int, percent float64, err error) type OnUploadProgressFunc func(read, total int, percent float64, err error)
type chunckSize int
func (c chunckSize) valOrDefault() int {
if c == 0 {
return defaultBlockSize
}
return int(c)
}
func (c chunckSize) toSizeT() C.size_t {
return C.size_t(c.valOrDefault())
}
type UploadOptions struct { type UploadOptions struct {
// filepath can be the full path when using UploadFile // Filepath can be the full path when using UploadFile
// otherwise the file name. // otherwise the file name.
// It is used to detect the mimetype. // It is used to detect the mimetype.
filepath string Filepath string
// chunkSize is the size of each upload chunk, passed as `blockSize` to the Codex node // ChunkSize is the size of each upload chunk, passed as `blockSize` to the Codex node
// store. Default is to 64 KB. // store. Default is to 64 KB.
chunkSize chunckSize ChunkSize ChunkSize
// onProgress is a callback function that is called after each chunk is uploaded with: // OnProgress is a callback function that is called after each chunk is uploaded with:
// - read: the number of bytes read in the last chunk. // - read: the number of bytes read in the last chunk.
// - total: the total number of bytes read so far. // - total: the total number of bytes read so far.
// - percent: the percentage of the total file size that has been uploaded. It is // - percent: the percentage of the total file size that has been uploaded. It is
@ -72,7 +58,7 @@ type UploadOptions struct {
// If the chunk size is more than the `chunkSize` parameter, the callback is called // If the chunk size is more than the `chunkSize` parameter, the callback is called
// after the block is actually stored in the block store. Otherwise, it is called // after the block is actually stored in the block store. Otherwise, it is called
// after the chunk is sent to the stream. // after the chunk is sent to the stream.
onProgress onUploadProgressFunc OnProgress OnUploadProgressFunc
} }
func getReaderSize(r io.Reader) int64 { func getReaderSize(r io.Reader) int64 {
@ -98,10 +84,10 @@ func (node CodexNode) UploadInit(options *UploadOptions) (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cFilename = C.CString(options.filepath) var cFilename = C.CString(options.Filepath)
defer C.free(unsafe.Pointer(cFilename)) defer C.free(unsafe.Pointer(cFilename))
if C.cGoCodexUploadInit(node.ctx, cFilename, options.chunkSize.toSizeT(), bridge.resp) != C.RET_OK { if C.cGoCodexUploadInit(node.ctx, cFilename, options.ChunkSize.toSizeT(), bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexUploadInit") return "", bridge.callError("cGoCodexUploadInit")
} }
@ -184,11 +170,11 @@ func (node CodexNode) UploadReader(options UploadOptions, r io.Reader) (string,
return "", err return "", err
} }
buf := make([]byte, options.chunkSize.valOrDefault()) buf := make([]byte, options.ChunkSize.valOrDefault())
total := 0 total := 0
var size int64 var size int64
if options.onProgress != nil { if options.OnProgress != nil {
size = getReaderSize(r) size = getReaderSize(r)
} }
@ -219,16 +205,16 @@ func (node CodexNode) UploadReader(options UploadOptions, r io.Reader) (string,
} }
total += n total += n
if options.onProgress != nil && size > 0 { if options.OnProgress != nil && size > 0 {
percent := float64(total) / float64(size) * 100.0 percent := float64(total) / float64(size) * 100.0
// The last block could be a bit over the size due to padding // The last block could be a bit over the size due to padding
// on the chunk size. // on the chunk size.
if percent > 100.0 { if percent > 100.0 {
percent = 100.0 percent = 100.0
} }
options.onProgress(n, total, percent, nil) options.OnProgress(n, total, percent, nil)
} else if options.onProgress != nil { } else if options.OnProgress != nil {
options.onProgress(n, total, 0, nil) options.OnProgress(n, total, 0, nil)
} }
} }
@ -267,8 +253,8 @@ func (node CodexNode) UploadFile(options UploadOptions) (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if options.onProgress != nil { if options.OnProgress != nil {
stat, err := os.Stat(options.filepath) stat, err := os.Stat(options.Filepath)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -290,7 +276,7 @@ func (node CodexNode) UploadFile(options UploadOptions) (string, error) {
percent = 100.0 percent = 100.0
} }
options.onProgress(read, int(size), percent, nil) options.OnProgress(read, int(size), percent, nil)
} }
} }
} }

View File

@ -16,7 +16,7 @@ func TestUploadReader(t *testing.T) {
buf := bytes.NewBuffer([]byte("Hello World!")) buf := bytes.NewBuffer([]byte("Hello World!"))
len := buf.Len() len := buf.Len()
cid, err := codex.UploadReader(UploadOptions{filepath: "hello.txt", onProgress: func(read, total int, percent float64, err error) { cid, err := codex.UploadReader(UploadOptions{Filepath: "hello.txt", OnProgress: func(read, total int, percent float64, err error) {
if err != nil { if err != nil {
log.Fatalf("Error happened during upload: %v\n", err) log.Fatalf("Error happened during upload: %v\n", err)
} }
@ -52,7 +52,7 @@ func TestUploadFile(t *testing.T) {
log.Fatalf("Error happened during file stat: %v\n", err) log.Fatalf("Error happened during file stat: %v\n", err)
} }
options := UploadOptions{filepath: "./testdata/hello.txt", onProgress: func(read, total int, percent float64, err error) { options := UploadOptions{Filepath: "./testdata/hello.txt", OnProgress: func(read, total int, percent float64, err error) {
if err != nil { if err != nil {
log.Fatalf("Error happened during upload: %v\n", err) log.Fatalf("Error happened during upload: %v\n", err)
} }
@ -82,7 +82,7 @@ func TestUploadFile(t *testing.T) {
func TestUploadFileNoProgress(t *testing.T) { func TestUploadFileNoProgress(t *testing.T) {
codex := newCodexNode(t) codex := newCodexNode(t)
options := UploadOptions{filepath: "./testdata/doesnt_exist.txt"} options := UploadOptions{Filepath: "./testdata/doesnt_exist.txt"}
cid, err := codex.UploadFile(options) cid, err := codex.UploadFile(options)
if err == nil { if err == nil {
@ -97,7 +97,7 @@ func TestUploadFileNoProgress(t *testing.T) {
func TestManualUpload(t *testing.T) { func TestManualUpload(t *testing.T) {
codex := newCodexNode(t) codex := newCodexNode(t)
sessionId, err := codex.UploadInit(&UploadOptions{filepath: "hello.txt"}) sessionId, err := codex.UploadInit(&UploadOptions{Filepath: "hello.txt"})
if err != nil { if err != nil {
log.Fatal("Error happened:", err.Error()) log.Fatal("Error happened:", err.Error())
} }