logos-storage/logos-storage-go-bindings
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-go-bindings.git synced 2026-01-02 13:33:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Export fields from download and upload structure

Browse Source
This commit is contained in:
Arnaud 2025-10-14 11:45:23 +02:00
parent 0ea52a8825
commit 773324d620
No known key found for this signature in database
GPG Key ID: 20E40A5D3110766F
8 changed files with 92 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/artifacts.yml vendored
View File

@ -83,7 +83,13 @@ jobs:
git config --global core.symlinks false
make update
make libcodex
- name: Make sure that tests pass
env:
CGO_ENABLED: 1
run: make
run: make test
- name: Package artifacts Linux
if: matrix.target.os == 'ubuntu-latest' || matrix.target.os == 'ubuntu-24.04-arm'
run: |
@ -129,6 +135,5 @@ jobs:
- name: Create release
uses: softprops/action-gh-release@v2
with:
body_path: ./RELEASE.md
files: dist/**
draft: true

6
CHANGELOG.md
View File

@ -1,3 +1,9 @@
## v0.0.14 (2025-10-14)
### Notes
- Export fields in upload and download struct
- Fix typo
## v0.0.13 (2025-10-14)
### Notes

14
codex/codex.go
View File

@ -190,6 +190,20 @@ type CodexNode struct {
ctx unsafe.Pointer
}
type ChunkSize int
func (c ChunkSize) valOrDefault() int {
if c == 0 {
return defaultBlockSize
}
return int(c)
}
func (c ChunkSize) toSizeT() C.size_t {
return C.size_t(c.valOrDefault())
}
// New creates a new Codex node with the provided configuration.
// The node is not started automatically; you need to call CodexStart
// to start it.

72
codex/download.go
View File

@ -31,58 +31,58 @@ import (
"unsafe"
)
type onDownloadProgressFunc func(read, total int, percent float64, err error)
type OnDownloadProgressFunc func(read, total int, percent float64, err error)
// DownloadStreamOptions is used to download a file
// in a streaming manner in Codex.
type DownloadStreamOptions = struct {
// filepath is the path destination used by DownloadStream.
// Filepath is the path destination used by DownloadStream.
// If it is set, the content will be written into the specified
// path.
filepath string
Filepath string
// chunkSize is the size of each downloaded chunk. Default is to 64 KB.
chunkSize chunckSize
// ChunkSize is the size of each downloaded chunk. Default is to 64 KB.
ChunkSize ChunkSize
// onProgress is a callback function that is called after each chunk is download with:
// OnProgress is a callback function that is called after each chunk is download with:
// - read: the number of bytes downloaded for the last chunk.
// - total: the total number of bytes downloaded so far.
// - percent: the percentage of the total file size that has been downloaded. It is
// determined from `datasetSize`.
// - err: an error, if one occurred.
onProgress onDownloadProgressFunc
OnProgress OnDownloadProgressFunc
// writer is the path destination used by DownloadStream.
// Writer is the path destination used by DownloadStream.
// If it is set, the content will be written into the specified
// writer.
writer io.Writer
// Writer.
Writer io.Writer
// local defines the way to download the content.
// Local defines the way to download the content.
// If true, the content will be downloaded from the
// local node.
// Local node.
// If false (default), the content will be downloaded
// from the network.
local bool
Local bool
// datasetSize is the total size of the dataset being downloaded.
datasetSize int
// DatasetSize is the total size of the dataset being downloaded.
DatasetSize int
// datasetSizeAuto if true, will fetch the manifest before starting
// DatasetSizeAuto if true, will fetch the manifest before starting
// the downloaded to retrive the size of the data.
datasetSizeAuto bool
DatasetSizeAuto bool
}
// DownloadInitOptions is used to create a download session.
type DownloadInitOptions = struct {
// local defines the way to download the content.
// Local defines the way to download the content.
// If true, the content will be downloaded from the
// local node.
// If false (default), the content will be downloaded
// from the network.
local bool
Local bool
// chunkSize is the size of each downloaded chunk. Default is to 64 KB.
chunkSize chunckSize
// ChunkSize is the size of each downloaded chunk. Default is to 64 KB.
ChunkSize ChunkSize
}
// Manifest is the object containing the information of
@ -149,14 +149,14 @@ func (node CodexNode) DownloadStream(cid string, options DownloadStreamOptions)
bridge := newBridgeCtx()
defer bridge.free()
if options.datasetSizeAuto {
if options.DatasetSizeAuto {
manifest, err := node.DownloadManifest(cid)
if err != nil {
return err
}
options.datasetSize = manifest.DatasetSize
options.DatasetSize = manifest.DatasetSize
}
total := 0
@ -165,36 +165,36 @@ func (node CodexNode) DownloadStream(cid string, options DownloadStreamOptions)
return
}
if options.writer != nil {
w := options.writer
if options.Writer != nil {
w := options.Writer
if _, err := w.Write(chunk); err != nil {
if options.onProgress != nil {
options.onProgress(0, 0, 0.0, err)
if options.OnProgress != nil {
options.OnProgress(0, 0, 0.0, err)
}
}
}
total += read
if options.onProgress != nil {
if options.OnProgress != nil {
var percent = 0.0
if options.datasetSize > 0 {
percent = float64(total) / float64(options.datasetSize) * 100.0
if options.DatasetSize > 0 {
percent = float64(total) / float64(options.DatasetSize) * 100.0
}
options.onProgress(read, total, percent, nil)
options.OnProgress(read, total, percent, nil)
}
}
var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid))
var cFilepath = C.CString(options.filepath)
var cFilepath = C.CString(options.Filepath)
defer C.free(unsafe.Pointer(cFilepath))
var cLocal = C.bool(options.local)
var cLocal = C.bool(options.Local)
if C.cGoCodexDownloadStream(node.ctx, cCid, options.chunkSize.toSizeT(), cLocal, cFilepath, bridge.resp) != C.RET_OK {
if C.cGoCodexDownloadStream(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, cFilepath, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexDownloadLocal")
}
@ -212,9 +212,9 @@ func (node CodexNode) DownloadInit(cid string, options DownloadInitOptions) erro
var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid))
var cLocal = C.bool(options.local)
var cLocal = C.bool(options.Local)
if C.cGoCodexDownloadInit(node.ctx, cCid, options.chunkSize.toSizeT(), cLocal, bridge.resp) != C.RET_OK {
if C.cGoCodexDownloadInit(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexDownloadInit")
}

12
codex/download_test.go
View File

@ -19,10 +19,10 @@ func TestDownloadStream(t *testing.T) {
totalBytes := 0
finalPercent := 0.0
opt := DownloadStreamOptions{
writer: f,
datasetSize: len,
filepath: "testdata/hello.downloaded.writer.txt",
onProgress: func(read, total int, percent float64, err error) {
Writer: f,
DatasetSize: len,
Filepath: "testdata/hello.downloaded.writer.txt",
OnProgress: func(read, total int, percent float64, err error) {
if err != nil {
t.Fatalf("Error happening during download: %v\n", err)
}
@ -61,8 +61,8 @@ func TestDownloadStreamWithAutosize(t *testing.T) {
totalBytes := 0
finalPercent := 0.0
opt := DownloadStreamOptions{
datasetSizeAuto: true,
onProgress: func(read, total int, percent float64, err error) {
DatasetSizeAuto: true,
OnProgress: func(read, total int, percent float64, err error) {
if err != nil {
t.Fatalf("Error happening during download: %v\n", err)
}

2
codex/testutil.go
View File

@ -65,7 +65,7 @@ func uploadHelper(t *testing.T, codex *CodexNode) (string, int) {
buf := bytes.NewBuffer([]byte("Hello World!"))
len := buf.Len()
cid, err := codex.UploadReader(UploadOptions{filepath: "hello.txt"}, buf)
cid, err := codex.UploadReader(UploadOptions{Filepath: "hello.txt"}, buf)
if err != nil {
t.Fatalf("Error happened during upload: %v\n", err)
}

50
codex/upload.go
View File

@ -35,33 +35,19 @@ import (
const defaultBlockSize = 1024 * 64
type onUploadProgressFunc func(read, total int, percent float64, err error)
type chunckSize int
func (c chunckSize) valOrDefault() int {
if c == 0 {
return defaultBlockSize
}
return int(c)
}
func (c chunckSize) toSizeT() C.size_t {
return C.size_t(c.valOrDefault())
}
type OnUploadProgressFunc func(read, total int, percent float64, err error)
type UploadOptions struct {
// filepath can be the full path when using UploadFile
// Filepath can be the full path when using UploadFile
// otherwise the file name.
// It is used to detect the mimetype.
filepath string
Filepath string
// chunkSize is the size of each upload chunk, passed as `blockSize` to the Codex node
// ChunkSize is the size of each upload chunk, passed as `blockSize` to the Codex node
// store. Default is to 64 KB.
chunkSize chunckSize
ChunkSize ChunkSize
// onProgress is a callback function that is called after each chunk is uploaded with:
// OnProgress is a callback function that is called after each chunk is uploaded with:
// - read: the number of bytes read in the last chunk.
// - total: the total number of bytes read so far.
// - percent: the percentage of the total file size that has been uploaded. It is
@ -72,7 +58,7 @@ type UploadOptions struct {
// If the chunk size is more than the `chunkSize` parameter, the callback is called
// after the block is actually stored in the block store. Otherwise, it is called
// after the chunk is sent to the stream.
onProgress onUploadProgressFunc
OnProgress OnUploadProgressFunc
}
func getReaderSize(r io.Reader) int64 {
@ -98,10 +84,10 @@ func (node CodexNode) UploadInit(options *UploadOptions) (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
var cFilename = C.CString(options.filepath)
var cFilename = C.CString(options.Filepath)
defer C.free(unsafe.Pointer(cFilename))
if C.cGoCodexUploadInit(node.ctx, cFilename, options.chunkSize.toSizeT(), bridge.resp) != C.RET_OK {
if C.cGoCodexUploadInit(node.ctx, cFilename, options.ChunkSize.toSizeT(), bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexUploadInit")
}
@ -184,11 +170,11 @@ func (node CodexNode) UploadReader(options UploadOptions, r io.Reader) (string,
return "", err
}
buf := make([]byte, options.chunkSize.valOrDefault())
buf := make([]byte, options.ChunkSize.valOrDefault())
total := 0
var size int64
if options.onProgress != nil {
if options.OnProgress != nil {
size = getReaderSize(r)
}
@ -219,16 +205,16 @@ func (node CodexNode) UploadReader(options UploadOptions, r io.Reader) (string,
}
total += n
if options.onProgress != nil && size > 0 {
if options.OnProgress != nil && size > 0 {
percent := float64(total) / float64(size) * 100.0
// The last block could be a bit over the size due to padding
// on the chunk size.
if percent > 100.0 {
percent = 100.0
}
options.onProgress(n, total, percent, nil)
} else if options.onProgress != nil {
options.onProgress(n, total, 0, nil)
options.OnProgress(n, total, percent, nil)
} else if options.OnProgress != nil {
options.OnProgress(n, total, 0, nil)
}
}
@ -267,8 +253,8 @@ func (node CodexNode) UploadFile(options UploadOptions) (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
if options.onProgress != nil {
stat, err := os.Stat(options.filepath)
if options.OnProgress != nil {
stat, err := os.Stat(options.Filepath)
if err != nil {
return "", err
}
@ -290,7 +276,7 @@ func (node CodexNode) UploadFile(options UploadOptions) (string, error) {
percent = 100.0
}
options.onProgress(read, int(size), percent, nil)
options.OnProgress(read, int(size), percent, nil)
}
}
}

8
codex/upload_test.go
View File

@ -16,7 +16,7 @@ func TestUploadReader(t *testing.T) {
buf := bytes.NewBuffer([]byte("Hello World!"))
len := buf.Len()
cid, err := codex.UploadReader(UploadOptions{filepath: "hello.txt", onProgress: func(read, total int, percent float64, err error) {
cid, err := codex.UploadReader(UploadOptions{Filepath: "hello.txt", OnProgress: func(read, total int, percent float64, err error) {
if err != nil {
log.Fatalf("Error happened during upload: %v\n", err)
}
@ -52,7 +52,7 @@ func TestUploadFile(t *testing.T) {
log.Fatalf("Error happened during file stat: %v\n", err)
}
options := UploadOptions{filepath: "./testdata/hello.txt", onProgress: func(read, total int, percent float64, err error) {
options := UploadOptions{Filepath: "./testdata/hello.txt", OnProgress: func(read, total int, percent float64, err error) {
if err != nil {
log.Fatalf("Error happened during upload: %v\n", err)
}
@ -82,7 +82,7 @@ func TestUploadFile(t *testing.T) {
func TestUploadFileNoProgress(t *testing.T) {
codex := newCodexNode(t)
options := UploadOptions{filepath: "./testdata/doesnt_exist.txt"}
options := UploadOptions{Filepath: "./testdata/doesnt_exist.txt"}
cid, err := codex.UploadFile(options)
if err == nil {
@ -97,7 +97,7 @@ func TestUploadFileNoProgress(t *testing.T) {
func TestManualUpload(t *testing.T) {
codex := newCodexNode(t)
sessionId, err := codex.UploadInit(&UploadOptions{filepath: "hello.txt"})
sessionId, err := codex.UploadInit(&UploadOptions{Filepath: "hello.txt"})
if err != nil {
log.Fatal("Error happened:", err.Error())
}