mirror of
https://github.com/logos-storage/logos-storage-go-bindings.git
synced 2026-01-04 06:23:06 +00:00
Export fields from download and upload structure
This commit is contained in:
parent
0ea52a8825
commit
773324d620
9
.github/workflows/artifacts.yml
vendored
9
.github/workflows/artifacts.yml
vendored
@ -83,7 +83,13 @@ jobs:
|
|||||||
git config --global core.symlinks false
|
git config --global core.symlinks false
|
||||||
make update
|
make update
|
||||||
make libcodex
|
make libcodex
|
||||||
|
|
||||||
|
- name: Make sure that tests pass
|
||||||
|
env:
|
||||||
|
CGO_ENABLED: 1
|
||||||
|
run: make
|
||||||
|
run: make test
|
||||||
|
|
||||||
- name: Package artifacts Linux
|
- name: Package artifacts Linux
|
||||||
if: matrix.target.os == 'ubuntu-latest' || matrix.target.os == 'ubuntu-24.04-arm'
|
if: matrix.target.os == 'ubuntu-latest' || matrix.target.os == 'ubuntu-24.04-arm'
|
||||||
run: |
|
run: |
|
||||||
@ -129,6 +135,5 @@ jobs:
|
|||||||
- name: Create release
|
- name: Create release
|
||||||
uses: softprops/action-gh-release@v2
|
uses: softprops/action-gh-release@v2
|
||||||
with:
|
with:
|
||||||
body_path: ./RELEASE.md
|
|
||||||
files: dist/**
|
files: dist/**
|
||||||
draft: true
|
draft: true
|
||||||
|
|||||||
@ -1,3 +1,9 @@
|
|||||||
|
## v0.0.14 (2025-10-14)
|
||||||
|
### Notes
|
||||||
|
|
||||||
|
- Export fields in upload and download struct
|
||||||
|
- Fix typo
|
||||||
|
|
||||||
## v0.0.13 (2025-10-14)
|
## v0.0.13 (2025-10-14)
|
||||||
### Notes
|
### Notes
|
||||||
|
|
||||||
|
|||||||
@ -190,6 +190,20 @@ type CodexNode struct {
|
|||||||
ctx unsafe.Pointer
|
ctx unsafe.Pointer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ChunkSize int
|
||||||
|
|
||||||
|
func (c ChunkSize) valOrDefault() int {
|
||||||
|
if c == 0 {
|
||||||
|
return defaultBlockSize
|
||||||
|
}
|
||||||
|
|
||||||
|
return int(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c ChunkSize) toSizeT() C.size_t {
|
||||||
|
return C.size_t(c.valOrDefault())
|
||||||
|
}
|
||||||
|
|
||||||
// New creates a new Codex node with the provided configuration.
|
// New creates a new Codex node with the provided configuration.
|
||||||
// The node is not started automatically; you need to call CodexStart
|
// The node is not started automatically; you need to call CodexStart
|
||||||
// to start it.
|
// to start it.
|
||||||
|
|||||||
@ -31,58 +31,58 @@ import (
|
|||||||
"unsafe"
|
"unsafe"
|
||||||
)
|
)
|
||||||
|
|
||||||
type onDownloadProgressFunc func(read, total int, percent float64, err error)
|
type OnDownloadProgressFunc func(read, total int, percent float64, err error)
|
||||||
|
|
||||||
// DownloadStreamOptions is used to download a file
|
// DownloadStreamOptions is used to download a file
|
||||||
// in a streaming manner in Codex.
|
// in a streaming manner in Codex.
|
||||||
type DownloadStreamOptions = struct {
|
type DownloadStreamOptions = struct {
|
||||||
// filepath is the path destination used by DownloadStream.
|
// Filepath is the path destination used by DownloadStream.
|
||||||
// If it is set, the content will be written into the specified
|
// If it is set, the content will be written into the specified
|
||||||
// path.
|
// path.
|
||||||
filepath string
|
Filepath string
|
||||||
|
|
||||||
// chunkSize is the size of each downloaded chunk. Default is to 64 KB.
|
// ChunkSize is the size of each downloaded chunk. Default is to 64 KB.
|
||||||
chunkSize chunckSize
|
ChunkSize ChunkSize
|
||||||
|
|
||||||
// onProgress is a callback function that is called after each chunk is download with:
|
// OnProgress is a callback function that is called after each chunk is download with:
|
||||||
// - read: the number of bytes downloaded for the last chunk.
|
// - read: the number of bytes downloaded for the last chunk.
|
||||||
// - total: the total number of bytes downloaded so far.
|
// - total: the total number of bytes downloaded so far.
|
||||||
// - percent: the percentage of the total file size that has been downloaded. It is
|
// - percent: the percentage of the total file size that has been downloaded. It is
|
||||||
// determined from `datasetSize`.
|
// determined from `datasetSize`.
|
||||||
// - err: an error, if one occurred.
|
// - err: an error, if one occurred.
|
||||||
onProgress onDownloadProgressFunc
|
OnProgress OnDownloadProgressFunc
|
||||||
|
|
||||||
// writer is the path destination used by DownloadStream.
|
// Writer is the path destination used by DownloadStream.
|
||||||
// If it is set, the content will be written into the specified
|
// If it is set, the content will be written into the specified
|
||||||
// writer.
|
// Writer.
|
||||||
writer io.Writer
|
Writer io.Writer
|
||||||
|
|
||||||
// local defines the way to download the content.
|
// Local defines the way to download the content.
|
||||||
// If true, the content will be downloaded from the
|
// If true, the content will be downloaded from the
|
||||||
// local node.
|
// Local node.
|
||||||
// If false (default), the content will be downloaded
|
// If false (default), the content will be downloaded
|
||||||
// from the network.
|
// from the network.
|
||||||
local bool
|
Local bool
|
||||||
|
|
||||||
// datasetSize is the total size of the dataset being downloaded.
|
// DatasetSize is the total size of the dataset being downloaded.
|
||||||
datasetSize int
|
DatasetSize int
|
||||||
|
|
||||||
// datasetSizeAuto if true, will fetch the manifest before starting
|
// DatasetSizeAuto if true, will fetch the manifest before starting
|
||||||
// the downloaded to retrive the size of the data.
|
// the downloaded to retrive the size of the data.
|
||||||
datasetSizeAuto bool
|
DatasetSizeAuto bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadInitOptions is used to create a download session.
|
// DownloadInitOptions is used to create a download session.
|
||||||
type DownloadInitOptions = struct {
|
type DownloadInitOptions = struct {
|
||||||
// local defines the way to download the content.
|
// Local defines the way to download the content.
|
||||||
// If true, the content will be downloaded from the
|
// If true, the content will be downloaded from the
|
||||||
// local node.
|
// local node.
|
||||||
// If false (default), the content will be downloaded
|
// If false (default), the content will be downloaded
|
||||||
// from the network.
|
// from the network.
|
||||||
local bool
|
Local bool
|
||||||
|
|
||||||
// chunkSize is the size of each downloaded chunk. Default is to 64 KB.
|
// ChunkSize is the size of each downloaded chunk. Default is to 64 KB.
|
||||||
chunkSize chunckSize
|
ChunkSize ChunkSize
|
||||||
}
|
}
|
||||||
|
|
||||||
// Manifest is the object containing the information of
|
// Manifest is the object containing the information of
|
||||||
@ -149,14 +149,14 @@ func (node CodexNode) DownloadStream(cid string, options DownloadStreamOptions)
|
|||||||
bridge := newBridgeCtx()
|
bridge := newBridgeCtx()
|
||||||
defer bridge.free()
|
defer bridge.free()
|
||||||
|
|
||||||
if options.datasetSizeAuto {
|
if options.DatasetSizeAuto {
|
||||||
manifest, err := node.DownloadManifest(cid)
|
manifest, err := node.DownloadManifest(cid)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
options.datasetSize = manifest.DatasetSize
|
options.DatasetSize = manifest.DatasetSize
|
||||||
}
|
}
|
||||||
|
|
||||||
total := 0
|
total := 0
|
||||||
@ -165,36 +165,36 @@ func (node CodexNode) DownloadStream(cid string, options DownloadStreamOptions)
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if options.writer != nil {
|
if options.Writer != nil {
|
||||||
w := options.writer
|
w := options.Writer
|
||||||
if _, err := w.Write(chunk); err != nil {
|
if _, err := w.Write(chunk); err != nil {
|
||||||
if options.onProgress != nil {
|
if options.OnProgress != nil {
|
||||||
options.onProgress(0, 0, 0.0, err)
|
options.OnProgress(0, 0, 0.0, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
total += read
|
total += read
|
||||||
|
|
||||||
if options.onProgress != nil {
|
if options.OnProgress != nil {
|
||||||
var percent = 0.0
|
var percent = 0.0
|
||||||
if options.datasetSize > 0 {
|
if options.DatasetSize > 0 {
|
||||||
percent = float64(total) / float64(options.datasetSize) * 100.0
|
percent = float64(total) / float64(options.DatasetSize) * 100.0
|
||||||
}
|
}
|
||||||
|
|
||||||
options.onProgress(read, total, percent, nil)
|
options.OnProgress(read, total, percent, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var cCid = C.CString(cid)
|
var cCid = C.CString(cid)
|
||||||
defer C.free(unsafe.Pointer(cCid))
|
defer C.free(unsafe.Pointer(cCid))
|
||||||
|
|
||||||
var cFilepath = C.CString(options.filepath)
|
var cFilepath = C.CString(options.Filepath)
|
||||||
defer C.free(unsafe.Pointer(cFilepath))
|
defer C.free(unsafe.Pointer(cFilepath))
|
||||||
|
|
||||||
var cLocal = C.bool(options.local)
|
var cLocal = C.bool(options.Local)
|
||||||
|
|
||||||
if C.cGoCodexDownloadStream(node.ctx, cCid, options.chunkSize.toSizeT(), cLocal, cFilepath, bridge.resp) != C.RET_OK {
|
if C.cGoCodexDownloadStream(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, cFilepath, bridge.resp) != C.RET_OK {
|
||||||
return bridge.callError("cGoCodexDownloadLocal")
|
return bridge.callError("cGoCodexDownloadLocal")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -212,9 +212,9 @@ func (node CodexNode) DownloadInit(cid string, options DownloadInitOptions) erro
|
|||||||
var cCid = C.CString(cid)
|
var cCid = C.CString(cid)
|
||||||
defer C.free(unsafe.Pointer(cCid))
|
defer C.free(unsafe.Pointer(cCid))
|
||||||
|
|
||||||
var cLocal = C.bool(options.local)
|
var cLocal = C.bool(options.Local)
|
||||||
|
|
||||||
if C.cGoCodexDownloadInit(node.ctx, cCid, options.chunkSize.toSizeT(), cLocal, bridge.resp) != C.RET_OK {
|
if C.cGoCodexDownloadInit(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, bridge.resp) != C.RET_OK {
|
||||||
return bridge.callError("cGoCodexDownloadInit")
|
return bridge.callError("cGoCodexDownloadInit")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -19,10 +19,10 @@ func TestDownloadStream(t *testing.T) {
|
|||||||
totalBytes := 0
|
totalBytes := 0
|
||||||
finalPercent := 0.0
|
finalPercent := 0.0
|
||||||
opt := DownloadStreamOptions{
|
opt := DownloadStreamOptions{
|
||||||
writer: f,
|
Writer: f,
|
||||||
datasetSize: len,
|
DatasetSize: len,
|
||||||
filepath: "testdata/hello.downloaded.writer.txt",
|
Filepath: "testdata/hello.downloaded.writer.txt",
|
||||||
onProgress: func(read, total int, percent float64, err error) {
|
OnProgress: func(read, total int, percent float64, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Error happening during download: %v\n", err)
|
t.Fatalf("Error happening during download: %v\n", err)
|
||||||
}
|
}
|
||||||
@ -61,8 +61,8 @@ func TestDownloadStreamWithAutosize(t *testing.T) {
|
|||||||
totalBytes := 0
|
totalBytes := 0
|
||||||
finalPercent := 0.0
|
finalPercent := 0.0
|
||||||
opt := DownloadStreamOptions{
|
opt := DownloadStreamOptions{
|
||||||
datasetSizeAuto: true,
|
DatasetSizeAuto: true,
|
||||||
onProgress: func(read, total int, percent float64, err error) {
|
OnProgress: func(read, total int, percent float64, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Error happening during download: %v\n", err)
|
t.Fatalf("Error happening during download: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -65,7 +65,7 @@ func uploadHelper(t *testing.T, codex *CodexNode) (string, int) {
|
|||||||
|
|
||||||
buf := bytes.NewBuffer([]byte("Hello World!"))
|
buf := bytes.NewBuffer([]byte("Hello World!"))
|
||||||
len := buf.Len()
|
len := buf.Len()
|
||||||
cid, err := codex.UploadReader(UploadOptions{filepath: "hello.txt"}, buf)
|
cid, err := codex.UploadReader(UploadOptions{Filepath: "hello.txt"}, buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Error happened during upload: %v\n", err)
|
t.Fatalf("Error happened during upload: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -35,33 +35,19 @@ import (
|
|||||||
|
|
||||||
const defaultBlockSize = 1024 * 64
|
const defaultBlockSize = 1024 * 64
|
||||||
|
|
||||||
type onUploadProgressFunc func(read, total int, percent float64, err error)
|
type OnUploadProgressFunc func(read, total int, percent float64, err error)
|
||||||
|
|
||||||
type chunckSize int
|
|
||||||
|
|
||||||
func (c chunckSize) valOrDefault() int {
|
|
||||||
if c == 0 {
|
|
||||||
return defaultBlockSize
|
|
||||||
}
|
|
||||||
|
|
||||||
return int(c)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c chunckSize) toSizeT() C.size_t {
|
|
||||||
return C.size_t(c.valOrDefault())
|
|
||||||
}
|
|
||||||
|
|
||||||
type UploadOptions struct {
|
type UploadOptions struct {
|
||||||
// filepath can be the full path when using UploadFile
|
// Filepath can be the full path when using UploadFile
|
||||||
// otherwise the file name.
|
// otherwise the file name.
|
||||||
// It is used to detect the mimetype.
|
// It is used to detect the mimetype.
|
||||||
filepath string
|
Filepath string
|
||||||
|
|
||||||
// chunkSize is the size of each upload chunk, passed as `blockSize` to the Codex node
|
// ChunkSize is the size of each upload chunk, passed as `blockSize` to the Codex node
|
||||||
// store. Default is to 64 KB.
|
// store. Default is to 64 KB.
|
||||||
chunkSize chunckSize
|
ChunkSize ChunkSize
|
||||||
|
|
||||||
// onProgress is a callback function that is called after each chunk is uploaded with:
|
// OnProgress is a callback function that is called after each chunk is uploaded with:
|
||||||
// - read: the number of bytes read in the last chunk.
|
// - read: the number of bytes read in the last chunk.
|
||||||
// - total: the total number of bytes read so far.
|
// - total: the total number of bytes read so far.
|
||||||
// - percent: the percentage of the total file size that has been uploaded. It is
|
// - percent: the percentage of the total file size that has been uploaded. It is
|
||||||
@ -72,7 +58,7 @@ type UploadOptions struct {
|
|||||||
// If the chunk size is more than the `chunkSize` parameter, the callback is called
|
// If the chunk size is more than the `chunkSize` parameter, the callback is called
|
||||||
// after the block is actually stored in the block store. Otherwise, it is called
|
// after the block is actually stored in the block store. Otherwise, it is called
|
||||||
// after the chunk is sent to the stream.
|
// after the chunk is sent to the stream.
|
||||||
onProgress onUploadProgressFunc
|
OnProgress OnUploadProgressFunc
|
||||||
}
|
}
|
||||||
|
|
||||||
func getReaderSize(r io.Reader) int64 {
|
func getReaderSize(r io.Reader) int64 {
|
||||||
@ -98,10 +84,10 @@ func (node CodexNode) UploadInit(options *UploadOptions) (string, error) {
|
|||||||
bridge := newBridgeCtx()
|
bridge := newBridgeCtx()
|
||||||
defer bridge.free()
|
defer bridge.free()
|
||||||
|
|
||||||
var cFilename = C.CString(options.filepath)
|
var cFilename = C.CString(options.Filepath)
|
||||||
defer C.free(unsafe.Pointer(cFilename))
|
defer C.free(unsafe.Pointer(cFilename))
|
||||||
|
|
||||||
if C.cGoCodexUploadInit(node.ctx, cFilename, options.chunkSize.toSizeT(), bridge.resp) != C.RET_OK {
|
if C.cGoCodexUploadInit(node.ctx, cFilename, options.ChunkSize.toSizeT(), bridge.resp) != C.RET_OK {
|
||||||
return "", bridge.callError("cGoCodexUploadInit")
|
return "", bridge.callError("cGoCodexUploadInit")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -184,11 +170,11 @@ func (node CodexNode) UploadReader(options UploadOptions, r io.Reader) (string,
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
buf := make([]byte, options.chunkSize.valOrDefault())
|
buf := make([]byte, options.ChunkSize.valOrDefault())
|
||||||
total := 0
|
total := 0
|
||||||
|
|
||||||
var size int64
|
var size int64
|
||||||
if options.onProgress != nil {
|
if options.OnProgress != nil {
|
||||||
size = getReaderSize(r)
|
size = getReaderSize(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -219,16 +205,16 @@ func (node CodexNode) UploadReader(options UploadOptions, r io.Reader) (string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
total += n
|
total += n
|
||||||
if options.onProgress != nil && size > 0 {
|
if options.OnProgress != nil && size > 0 {
|
||||||
percent := float64(total) / float64(size) * 100.0
|
percent := float64(total) / float64(size) * 100.0
|
||||||
// The last block could be a bit over the size due to padding
|
// The last block could be a bit over the size due to padding
|
||||||
// on the chunk size.
|
// on the chunk size.
|
||||||
if percent > 100.0 {
|
if percent > 100.0 {
|
||||||
percent = 100.0
|
percent = 100.0
|
||||||
}
|
}
|
||||||
options.onProgress(n, total, percent, nil)
|
options.OnProgress(n, total, percent, nil)
|
||||||
} else if options.onProgress != nil {
|
} else if options.OnProgress != nil {
|
||||||
options.onProgress(n, total, 0, nil)
|
options.OnProgress(n, total, 0, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -267,8 +253,8 @@ func (node CodexNode) UploadFile(options UploadOptions) (string, error) {
|
|||||||
bridge := newBridgeCtx()
|
bridge := newBridgeCtx()
|
||||||
defer bridge.free()
|
defer bridge.free()
|
||||||
|
|
||||||
if options.onProgress != nil {
|
if options.OnProgress != nil {
|
||||||
stat, err := os.Stat(options.filepath)
|
stat, err := os.Stat(options.Filepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@ -290,7 +276,7 @@ func (node CodexNode) UploadFile(options UploadOptions) (string, error) {
|
|||||||
percent = 100.0
|
percent = 100.0
|
||||||
}
|
}
|
||||||
|
|
||||||
options.onProgress(read, int(size), percent, nil)
|
options.OnProgress(read, int(size), percent, nil)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -16,7 +16,7 @@ func TestUploadReader(t *testing.T) {
|
|||||||
|
|
||||||
buf := bytes.NewBuffer([]byte("Hello World!"))
|
buf := bytes.NewBuffer([]byte("Hello World!"))
|
||||||
len := buf.Len()
|
len := buf.Len()
|
||||||
cid, err := codex.UploadReader(UploadOptions{filepath: "hello.txt", onProgress: func(read, total int, percent float64, err error) {
|
cid, err := codex.UploadReader(UploadOptions{Filepath: "hello.txt", OnProgress: func(read, total int, percent float64, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Error happened during upload: %v\n", err)
|
log.Fatalf("Error happened during upload: %v\n", err)
|
||||||
}
|
}
|
||||||
@ -52,7 +52,7 @@ func TestUploadFile(t *testing.T) {
|
|||||||
log.Fatalf("Error happened during file stat: %v\n", err)
|
log.Fatalf("Error happened during file stat: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
options := UploadOptions{filepath: "./testdata/hello.txt", onProgress: func(read, total int, percent float64, err error) {
|
options := UploadOptions{Filepath: "./testdata/hello.txt", OnProgress: func(read, total int, percent float64, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Error happened during upload: %v\n", err)
|
log.Fatalf("Error happened during upload: %v\n", err)
|
||||||
}
|
}
|
||||||
@ -82,7 +82,7 @@ func TestUploadFile(t *testing.T) {
|
|||||||
func TestUploadFileNoProgress(t *testing.T) {
|
func TestUploadFileNoProgress(t *testing.T) {
|
||||||
codex := newCodexNode(t)
|
codex := newCodexNode(t)
|
||||||
|
|
||||||
options := UploadOptions{filepath: "./testdata/doesnt_exist.txt"}
|
options := UploadOptions{Filepath: "./testdata/doesnt_exist.txt"}
|
||||||
|
|
||||||
cid, err := codex.UploadFile(options)
|
cid, err := codex.UploadFile(options)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@ -97,7 +97,7 @@ func TestUploadFileNoProgress(t *testing.T) {
|
|||||||
func TestManualUpload(t *testing.T) {
|
func TestManualUpload(t *testing.T) {
|
||||||
codex := newCodexNode(t)
|
codex := newCodexNode(t)
|
||||||
|
|
||||||
sessionId, err := codex.UploadInit(&UploadOptions{filepath: "hello.txt"})
|
sessionId, err := codex.UploadInit(&UploadOptions{Filepath: "hello.txt"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal("Error happened:", err.Error())
|
log.Fatal("Error happened:", err.Error())
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user